From 84c9ab0e3d284eb9ed93db8a2e45bb42b2554873 Mon Sep 17 00:00:00 2001 From: bramhanandlingala Date: Tue, 21 Jul 2026 00:02:00 +0530 Subject: [PATCH] fix(bitbucket): paginate remote-scopes API to return all repositories (#8999) --- backend/plugins/bitbucket/api/remote_api.go | 14 +++- .../plugins/bitbucket/api/remote_api_test.go | 83 +++++++++++++++++++ backend/plugins/bitbucket/models/repo.go | 2 + 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/bitbucket/api/remote_api_test.go diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index cf60b1bd51d..36fa45a0de8 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -107,6 +107,12 @@ func listBitbucketWorkspaces( FullName: r.GroupName(), }) } + if resBody.Next != "" { + nextPage = &BitbucketRemotePagination{ + Page: page.Page + 1, + PageLen: page.PageLen, + } + } return } @@ -123,7 +129,7 @@ func listBitbucketRepos( var res *http.Response // list projects part res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", workspace), url.Values{ - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size,next"}, "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, nil) @@ -153,6 +159,12 @@ func listBitbucketRepos( Data: r.ConvertApiScope(), }) } + if resBody.Next != "" { + nextPage = &BitbucketRemotePagination{ + Page: page.Page + 1, + PageLen: page.PageLen, + } + } return } diff --git a/backend/plugins/bitbucket/api/remote_api_test.go b/backend/plugins/bitbucket/api/remote_api_test.go new file mode 100644 index 00000000000..26bdbc4a2a4 --- /dev/null +++ b/backend/plugins/bitbucket/api/remote_api_test.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +type fakeApiClient struct{ body string } + +func (f *fakeApiClient) SetData(name string, data interface{}) {} +func (f *fakeApiClient) GetData(name string) interface{} { return nil } +func (f *fakeApiClient) SetHeaders(headers map[string]string) {} +func (f *fakeApiClient) SetBeforeFunction(callback plugin.ApiClientBeforeRequest) {} +func (f *fakeApiClient) GetBeforeFunction() plugin.ApiClientBeforeRequest { return nil } +func (f *fakeApiClient) SetAfterFunction(callback plugin.ApiClientAfterResponse) {} +func (f *fakeApiClient) GetAfterFunction() plugin.ApiClientAfterResponse { return nil } + +func (f *fakeApiClient) Get(path string, query url.Values, headers http.Header) (*http.Response, errors.Error) { + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(f.body))}, nil +} +func (f *fakeApiClient) Post(path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error) { + return nil, nil +} + +func TestListBitbucketRepos_ReturnsNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":2,"page":1,"size":4, + "next":"https://api.bitbucket.org/2.0/repositories/myworkspace?page=2", + "values":[{"name":"repo-a","full_name":"myworkspace/repo-a"}, + {"name":"repo-b","full_name":"myworkspace/repo-b"}]}`} + children, nextPage, err := listBitbucketRepos(client, "myworkspace", BitbucketRemotePagination{Page: 1, PageLen: 2}) + assert.Nil(t, err) + assert.Len(t, children, 2) + if assert.NotNil(t, nextPage) { + assert.Equal(t, 2, nextPage.Page) + } +} + +func TestListBitbucketRepos_LastPageHasNoNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":2,"page":2,"size":4, + "values":[{"name":"repo-c","full_name":"myworkspace/repo-c"}, + {"name":"repo-d","full_name":"myworkspace/repo-d"}]}`} + children, nextPage, err := listBitbucketRepos(client, "myworkspace", BitbucketRemotePagination{Page: 2, PageLen: 2}) + assert.Nil(t, err) + assert.Len(t, children, 2) + assert.Nil(t, nextPage) +} + +func TestListBitbucketWorkspaces_ReturnsNextPage(t *testing.T) { + client := &fakeApiClient{body: `{"pagelen":1,"page":1,"size":2, + "next":"https://api.bitbucket.org/2.0/user/workspaces?page=2", + "values":[{"workspace":{"slug":"ws-a","name":"Workspace A"}}]}`} + children, nextPage, err := listBitbucketWorkspaces(client, BitbucketRemotePagination{Page: 1, PageLen: 1}) + assert.Nil(t, err) + assert.Len(t, children, 1) + if assert.NotNil(t, nextPage) { + assert.Equal(t, 2, nextPage.Page) + } +} diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index b33df3615b6..126990a94ab 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -114,6 +114,7 @@ type WorkspaceResponse struct { Pagelen int `json:"pagelen"` Page int `json:"page"` Size int `json:"size"` + Next string `json:"next"` Values []GroupResponse `json:"values"` } @@ -142,6 +143,7 @@ type ReposResponse struct { Pagelen int `json:"pagelen"` Page int `json:"page"` Size int `json:"size"` + Next string `json:"next"` Values []BitbucketApiRepo `json:"values"` }