From 98dcef4f101fb4c8d41e7b92a2f21b9a3fd076d5 Mon Sep 17 00:00:00 2001 From: Evan Lynch Date: Mon, 20 Jul 2026 18:39:56 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(clickup):=20scaffold=20ClickUp=20data-?= =?UTF-8?q?source=20plugin=20(task=E2=86=92issue=20MVP)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new DevLake data-source plugin for ClickUp, mirroring the linear (structure) and trello (REST client) tracker plugins. - models: ClickUpConnection (token auth, no Bearer prefix), ClickUpList scope (keyed by list id), ClickUpTask, ClickUpUser, ClickUpTaskComment (stub), ClickUpScopeConfig (status + type mapping) - migrationscripts: register + addInitTables creating all _tool_clickup_* tables via migrationhelper.AutoMigrateTables - tasks: task/user collectors (page-based + member REST), extractors, convertors to ticket.Issue/BoardIssue/IssueAssignee, crossdomain.Account and ticket.Board; shared status/type/time helpers - api: connection CRUD + test-connection (GET /team), scope, scope-config, hierarchical remote-scopes (Team→Space→Folder→List), proxy, blueprint v200 - impl: all required plugin interfaces incl. GetTablesInfo listing every model, dependency-ordered SubTaskMetas and PrepareTaskData - register clickup in plugins/table_info_test.go go build ./plugins/clickup/... and go vet pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/plugins/clickup/api/blueprint_v200.go | 99 ++++++++ backend/plugins/clickup/api/connection_api.go | 175 ++++++++++++++ backend/plugins/clickup/api/init.go | 51 ++++ backend/plugins/clickup/api/remote_api.go | 189 +++++++++++++++ backend/plugins/clickup/api/scope_api.go | 105 +++++++++ .../plugins/clickup/api/scope_config_api.go | 106 +++++++++ backend/plugins/clickup/clickup.go | 43 ++++ backend/plugins/clickup/impl/impl.go | 220 ++++++++++++++++++ backend/plugins/clickup/models/connection.go | 74 ++++++ backend/plugins/clickup/models/list.go | 69 ++++++ .../20260720_add_init_tables.go | 47 ++++ .../migrationscripts/archived/models.go | 113 +++++++++ .../models/migrationscripts/register.go | 29 +++ .../plugins/clickup/models/scope_config.go | 56 +++++ backend/plugins/clickup/models/task.go | 52 +++++ .../plugins/clickup/models/task_comment.go | 45 ++++ backend/plugins/clickup/models/user.go | 37 +++ backend/plugins/clickup/tasks/api_client.go | 47 ++++ .../plugins/clickup/tasks/board_convertor.go | 92 ++++++++ backend/plugins/clickup/tasks/shared.go | 172 ++++++++++++++ .../plugins/clickup/tasks/task_collector.go | 111 +++++++++ .../plugins/clickup/tasks/task_convertor.go | 143 ++++++++++++ backend/plugins/clickup/tasks/task_data.go | 46 ++++ .../plugins/clickup/tasks/task_extractor.go | 152 ++++++++++++ .../plugins/clickup/tasks/user_collector.go | 72 ++++++ .../plugins/clickup/tasks/user_convertor.go | 88 +++++++ .../plugins/clickup/tasks/user_extractor.go | 83 +++++++ backend/plugins/table_info_test.go | 2 + 28 files changed, 2518 insertions(+) create mode 100644 backend/plugins/clickup/api/blueprint_v200.go create mode 100644 backend/plugins/clickup/api/connection_api.go create mode 100644 backend/plugins/clickup/api/init.go create mode 100644 backend/plugins/clickup/api/remote_api.go create mode 100644 backend/plugins/clickup/api/scope_api.go create mode 100644 backend/plugins/clickup/api/scope_config_api.go create mode 100644 backend/plugins/clickup/clickup.go create mode 100644 backend/plugins/clickup/impl/impl.go create mode 100644 backend/plugins/clickup/models/connection.go create mode 100644 backend/plugins/clickup/models/list.go create mode 100644 backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go create mode 100644 backend/plugins/clickup/models/migrationscripts/archived/models.go create mode 100644 backend/plugins/clickup/models/migrationscripts/register.go create mode 100644 backend/plugins/clickup/models/scope_config.go create mode 100644 backend/plugins/clickup/models/task.go create mode 100644 backend/plugins/clickup/models/task_comment.go create mode 100644 backend/plugins/clickup/models/user.go create mode 100644 backend/plugins/clickup/tasks/api_client.go create mode 100644 backend/plugins/clickup/tasks/board_convertor.go create mode 100644 backend/plugins/clickup/tasks/shared.go create mode 100644 backend/plugins/clickup/tasks/task_collector.go create mode 100644 backend/plugins/clickup/tasks/task_convertor.go create mode 100644 backend/plugins/clickup/tasks/task_data.go create mode 100644 backend/plugins/clickup/tasks/task_extractor.go create mode 100644 backend/plugins/clickup/tasks/user_collector.go create mode 100644 backend/plugins/clickup/tasks/user_convertor.go create mode 100644 backend/plugins/clickup/tasks/user_extractor.go diff --git a/backend/plugins/clickup/api/blueprint_v200.go b/backend/plugins/clickup/api/blueprint_v200.go new file mode 100644 index 00000000000..b2c02162bee --- /dev/null +++ b/backend/plugins/clickup/api/blueprint_v200.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig], + connection *models.ClickUpConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "clickup", + subtaskMetas, + scopeConfig.Entities, + tasks.ClickUpOptions{ + ConnectionId: connection.ID, + ListId: scope.ListId, + ScopeConfigId: scope.ScopeConfigId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig], + connection *models.ClickUpConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.ListId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/clickup/api/connection_api.go b/backend/plugins/clickup/api/connection_api.go new file mode 100644 index 00000000000..052b1e5f68a --- /dev/null +++ b/backend/plugins/clickup/api/connection_api.go @@ -0,0 +1,175 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type ClickUpTestConnResponse struct { + shared.ApiBody + Connection *models.ClickUpConn +} + +func testConnection(ctx context.Context, connection models.ClickUpConn) (*ClickUpTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = tasks.DefaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // GET /team lists the authenticated user's workspaces; it verifies the token. + res, err := apiClient.Get("team", nil, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API token") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := ClickUpTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test clickup connection +// @Summary test clickup connection +// @Description Test clickup Connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConn true "json body" +// @Success 200 {object} ClickUpTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.ClickUpConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test clickup connection by ID +// @Summary test clickup connection +// @Description Test clickup Connection +// @Tags plugins/clickup +// @Param connectionId path int true "connection ID" +// @Success 200 {object} ClickUpTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.ClickUpConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create clickup connection +// @Summary create clickup connection +// @Description Create clickup connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConnection true "json body" +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch clickup connection +// @Summary patch clickup connection +// @Description Patch clickup connection +// @Tags plugins/clickup +// @Param body body models.ClickUpConnection true "json body" +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a clickup connection +// @Summary delete a clickup connection +// @Description Delete a clickup connection +// @Tags plugins/clickup +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all clickup connections +// @Summary get all clickup connections +// @Description Get all clickup connections +// @Tags plugins/clickup +// @Success 200 {object} []models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get clickup connection detail +// @Summary get clickup connection detail +// @Description Get clickup connection detail +// @Tags plugins/clickup +// @Success 200 {object} models.ClickUpConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/clickup/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/clickup/api/init.go b/backend/plugins/clickup/api/init.go new file mode 100644 index 00000000000..0f484165433 --- /dev/null +++ b/backend/plugins/clickup/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.ClickUpConnection, models.ClickUpList, models.ClickUpScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.ClickUpConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpList, ClickUpRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.ClickUpConnection, models.ClickUpList, models.ClickUpScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.ClickUpConnection) models.ClickUpConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.ClickUpConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpList, ClickUpRemotePagination](raProxy, listClickUpRemoteScopes) +} diff --git a/backend/plugins/clickup/api/remote_api.go b/backend/plugins/clickup/api/remote_api.go new file mode 100644 index 00000000000..5734020d30b --- /dev/null +++ b/backend/plugins/clickup/api/remote_api.go @@ -0,0 +1,189 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// ClickUpRemotePagination is a placeholder: the ClickUp v2 hierarchy endpoints +// used here (team/space/folder/list) are not paginated, so there is never a +// next page. It exists to satisfy the DsRemoteApiScopeListHelper generic. +type ClickUpRemotePagination struct{} + +// groupId prefixes encode which level of the ClickUp hierarchy a group entry +// points at, so a single list function can walk Team -> Space -> Folder -> List. +const ( + groupTeamPrefix = "team:" + groupSpacePrefix = "space:" + groupFolderPrefix = "folder:" +) + +type clickUpNamedEntities struct { + Teams []clickUpNamedEntity `json:"teams"` + Spaces []clickUpNamedEntity `json:"spaces"` + Folders []clickUpNamedEntity `json:"folders"` + Lists []clickUpNamedEntity `json:"lists"` +} + +type clickUpNamedEntity struct { + Id string `json:"id"` + Name string `json:"name"` +} + +// listClickUpRemoteScopes walks the ClickUp hierarchy one level at a time. The +// config UI drives it via `groupId`: +// +// "" -> workspaces (Team) as groups +// team:{id} -> spaces as groups +// space:{id} -> folders as groups + folderless lists as selectable scopes +// folder:{id} -> lists as selectable scopes +func listClickUpRemoteScopes( + _ *models.ClickUpConnection, + apiClient plugin.ApiClient, + groupId string, + _ ClickUpRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], + nextPage *ClickUpRemotePagination, + err errors.Error, +) { + switch { + case groupId == "": + return listTeamsAsGroups(apiClient) + case strings.HasPrefix(groupId, groupTeamPrefix): + return listSpacesAsGroups(apiClient, strings.TrimPrefix(groupId, groupTeamPrefix)) + case strings.HasPrefix(groupId, groupSpacePrefix): + return listSpaceChildren(apiClient, strings.TrimPrefix(groupId, groupSpacePrefix)) + case strings.HasPrefix(groupId, groupFolderPrefix): + return listFolderLists(apiClient, strings.TrimPrefix(groupId, groupFolderPrefix)) + default: + return nil, nil, errors.BadInput.New(fmt.Sprintf("unrecognized groupId %q", groupId)) + } +} + +func getEntities(apiClient plugin.ApiClient, path string) (*clickUpNamedEntities, errors.Error) { + res, err := apiClient.Get(path, nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query ClickUp "+path) + } + var body clickUpNamedEntities + if err := api.UnmarshalResponse(res, &body); err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal ClickUp "+path+" response") + } + return &body, nil +} + +func groupEntry(id, name string) dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList] { + return dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: nil, + Id: id, + Name: name, + FullName: name, + } +} + +func listTeamsAsGroups(apiClient plugin.ApiClient) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, "team") + if err != nil { + return nil, nil, err + } + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(body.Teams)) + for _, team := range body.Teams { + children = append(children, groupEntry(groupTeamPrefix+team.Id, team.Name)) + } + return children, nil, nil +} + +func listSpacesAsGroups(apiClient plugin.ApiClient, teamId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, fmt.Sprintf("team/%s/space", teamId)) + if err != nil { + return nil, nil, err + } + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(body.Spaces)) + for _, space := range body.Spaces { + children = append(children, groupEntry(groupSpacePrefix+space.Id, space.Name)) + } + return children, nil, nil +} + +func listSpaceChildren(apiClient plugin.ApiClient, spaceId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { + folders, err := getEntities(apiClient, fmt.Sprintf("space/%s/folder", spaceId)) + if err != nil { + return nil, nil, err + } + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0) + for _, folder := range folders.Folders { + children = append(children, groupEntry(groupFolderPrefix+folder.Id, folder.Name)) + } + // Folderless lists live directly under the space. + lists, err := getEntities(apiClient, fmt.Sprintf("space/%s/list", spaceId)) + if err != nil { + return nil, nil, err + } + children = append(children, listsToScopeEntries(lists.Lists, spaceId)...) + return children, nil, nil +} + +func listFolderLists(apiClient plugin.ApiClient, folderId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, fmt.Sprintf("folder/%s/list", folderId)) + if err != nil { + return nil, nil, err + } + return listsToScopeEntries(body.Lists, ""), nil, nil +} + +// listsToScopeEntries maps ClickUp lists into selectable (leaf) scope entries. +func listsToScopeEntries(lists []clickUpNamedEntity, spaceId string) []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList] { + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(lists)) + for _, list := range lists { + list := list + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: list.Id, + Name: list.Name, + FullName: list.Name, + Data: &models.ClickUpList{ + ListId: list.Id, + Name: list.Name, + SpaceId: spaceId, + }, + }) + } + return entries +} + +// RemoteScopes lists the ClickUp lists available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the ClickUp API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/clickup/api/scope_api.go b/backend/plugins/clickup/api/scope_api.go new file mode 100644 index 00000000000..5cc7d012ad8 --- /dev/null +++ b/backend/plugins/clickup/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.ClickUpList] +type ScopeDetail api.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig] + +// PutScopes create or update clickup lists +// @Summary create or update clickup lists +// @Description Create or update clickup lists +// @Tags plugins/clickup +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.ClickUpList +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to clickup list +// @Summary patch to clickup list +// @Description patch to clickup list +// @Tags plugins/clickup +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "list ID" +// @Param scope body models.ClickUpList true "json" +// @Success 200 {object} models.ClickUpList +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get clickup lists +// @Summary get clickup lists +// @Description get clickup lists +// @Tags plugins/clickup +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one clickup list +// @Summary get one clickup list +// @Description get one clickup list +// @Tags plugins/clickup +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "list ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/clickup +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/clickup/api/scope_config_api.go b/backend/plugins/clickup/api/scope_config_api.go new file mode 100644 index 00000000000..fab37352f5c --- /dev/null +++ b/backend/plugins/clickup/api/scope_config_api.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for ClickUp +// @Summary create scope config for ClickUp +// @Description create scope config for ClickUp +// @Tags plugins/clickup +// @Accept application/json +// @Param scopeConfig body models.ClickUpScopeConfig true "scope config" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for ClickUp +// @Summary update scope config for ClickUp +// @Description update scope config for ClickUp +// @Tags plugins/clickup +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.ClickUpScopeConfig true "scope config" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/clickup +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.ClickUpScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig return projects details related by scope config +// @Summary return all related projects +// @Description return all related projects +// @Tags plugins/clickup +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ProjectScopeOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/clickup/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/clickup/clickup.go b/backend/plugins/clickup/clickup.go new file mode 100644 index 00000000000..a7806a1c7d7 --- /dev/null +++ b/backend/plugins/clickup/clickup.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/clickup/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.ClickUp //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "clickup"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "clickup connection id") + listId := cmd.Flags().StringP("list", "l", "", "clickup list id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("list") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "listId": *listId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/clickup/impl/impl.go b/backend/plugins/clickup/impl/impl.go new file mode 100644 index 00000000000..d578392f90b --- /dev/null +++ b/backend/plugins/clickup/impl/impl.go @@ -0,0 +1,220 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" + "github.com/apache/incubator-devlake/plugins/clickup/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/clickup/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*ClickUp)(nil) + +type ClickUp struct{} + +func (p ClickUp) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p ClickUp) Description() string { + return "To collect and enrich data from ClickUp" +} + +func (p ClickUp) Name() string { + return "clickup" +} + +func (p ClickUp) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/clickup" +} + +func (p ClickUp) Connection() dal.Tabler { + return &models.ClickUpConnection{} +} + +func (p ClickUp) Scope() plugin.ToolLayerScope { + return &models.ClickUpList{} +} + +func (p ClickUp) ScopeConfig() dal.Tabler { + return &models.ClickUpScopeConfig{} +} + +func (p ClickUp) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +// GetTablesInfo MUST list every model (CI `Test_GetPluginTablesInfo` fails otherwise). +func (p ClickUp) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.ClickUpConnection{}, + &models.ClickUpList{}, + &models.ClickUpScopeConfig{}, + &models.ClickUpUser{}, + &models.ClickUpTask{}, + &models.ClickUpTaskComment{}, + } +} + +// SubTaskMetas lists subtasks in dependency order: collect/extract before +// convert; users before tasks (issues reference accounts); lists (boards) +// before board_issues. +func (p ClickUp) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectUserMeta, + tasks.ExtractUserMeta, + tasks.CollectTaskMeta, + tasks.ExtractTaskMeta, + tasks.ConvertListMeta, + tasks.ConvertUserMeta, + tasks.ConvertTaskMeta, + } +} + +func (p ClickUp) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.ClickUpOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode ClickUp options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("clickup connectionId is invalid") + } + if op.ListId == "" { + return nil, errors.BadInput.New("clickup listId is required") + } + + connection := &models.ClickUpConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for ClickUp plugin") + } + + apiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create ClickUp API client") + } + + // Resolve the scope config. Default to an empty (non-nil) config so subtasks + // can rely on it being present. + scopeConfig := &models.ClickUpScopeConfig{} + if op.ScopeConfigId != 0 { + if err := taskCtx.GetDal().First(scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); err != nil { + return nil, errors.Default.Wrap(err, "error getting scope config for ClickUp plugin") + } + } + + taskData := &tasks.ClickUpTaskData{ + Options: &op, + ApiClient: apiClient, + ScopeConfig: scopeConfig, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p ClickUp) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p ClickUp) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p ClickUp) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.ClickUpTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.ApiClient != nil { + data.ApiClient.Release() + } + return nil +} diff --git a/backend/plugins/clickup/models/connection.go b/backend/plugins/clickup/models/connection.go new file mode 100644 index 00000000000..0f985983966 --- /dev/null +++ b/backend/plugins/clickup/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// ClickUpConn holds the essential information to connect to the ClickUp API. +// ClickUp authenticates with a personal API token passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. OAuth2 is out of +// scope for now. +type ClickUpConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the ClickUp API. +func (cc *ClickUpConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", cc.Token) + return nil +} + +func (cc *ClickUpConn) Sanitize() ClickUpConn { + cc.Token = utils.SanitizeString(cc.Token) + return *cc +} + +// ClickUpConnection holds ClickUpConn plus ID/Name for database storage. +type ClickUpConnection struct { + helper.BaseConnection `mapstructure:",squash"` + ClickUpConn `mapstructure:",squash"` +} + +func (connection ClickUpConnection) Sanitize() ClickUpConnection { + connection.ClickUpConn = connection.ClickUpConn.Sanitize() + return connection +} + +func (connection *ClickUpConnection) MergeFromRequest(target *ClickUpConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (ClickUpConnection) TableName() string { + return "_tool_clickup_connections" +} diff --git a/backend/plugins/clickup/models/list.go b/backend/plugins/clickup/models/list.go new file mode 100644 index 00000000000..5b230d91d14 --- /dev/null +++ b/backend/plugins/clickup/models/list.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*ClickUpList)(nil) + +// ClickUpList is the data-source scope for the ClickUp plugin. A ClickUp List +// owns tasks (analogous to a Jira board), mapping cleanly to a DevLake +// domain-layer ticket.Board. +type ClickUpList struct { + common.Scope `mapstructure:",squash"` + ListId string `json:"listId" mapstructure:"listId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + SpaceId string `json:"spaceId" mapstructure:"spaceId" gorm:"type:varchar(255)"` + SpaceName string `json:"spaceName" mapstructure:"spaceName" gorm:"type:varchar(255)"` +} + +func (l ClickUpList) ScopeId() string { + return l.ListId +} + +func (l ClickUpList) ScopeName() string { + return l.Name +} + +func (l ClickUpList) ScopeFullName() string { + if l.SpaceName != "" { + return l.SpaceName + "/" + l.Name + } + return l.Name +} + +func (l ClickUpList) ScopeParams() interface{} { + return &ClickUpApiParams{ + ConnectionId: l.ConnectionId, + ListId: l.ListId, + } +} + +func (ClickUpList) TableName() string { + return "_tool_clickup_lists" +} + +// ClickUpApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_clickup_* table. +type ClickUpApiParams struct { + ConnectionId uint64 + ListId string +} diff --git a/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go b/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go new file mode 100644 index 00000000000..24d4beb258e --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/20260720_add_init_tables.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/clickup/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.ClickUpConnection{}, + &archived.ClickUpList{}, + &archived.ClickUpScopeConfig{}, + &archived.ClickUpUser{}, + &archived.ClickUpTask{}, + &archived.ClickUpTaskComment{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260720000001 +} + +func (*addInitTables) Name() string { + return "clickup init schemas" +} diff --git a/backend/plugins/clickup/models/migrationscripts/archived/models.go b/backend/plugins/clickup/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..2ecc04f84d5 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/archived/models.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/clickup/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type ClickUpConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (ClickUpConnection) TableName() string { return "_tool_clickup_connections" } + +type ClickUpList struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + ListId string `json:"listId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + SpaceId string `json:"spaceId" gorm:"type:varchar(255)"` + SpaceName string `json:"spaceName" gorm:"type:varchar(255)"` +} + +func (ClickUpList) TableName() string { return "_tool_clickup_lists" } + +type ClickUpScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + IssueStatusTodo []string `json:"issueStatusTodo" gorm:"type:json;serializer:json"` + IssueStatusInProgress []string `json:"issueStatusInProgress" gorm:"type:json;serializer:json"` + IssueStatusDone []string `json:"issueStatusDone" gorm:"type:json;serializer:json"` + IssueTypeRequirement string `json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (ClickUpScopeConfig) TableName() string { return "_tool_clickup_scope_configs" } + +type ClickUpUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Username string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + Color string `gorm:"type:varchar(50)"` + ProfilePicture string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (ClickUpUser) TableName() string { return "_tool_clickup_users" } + +type ClickUpTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + ListId string `gorm:"index;type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + CustomId string `gorm:"type:varchar(255)"` + Name string + Description string + Status string `gorm:"type:varchar(255)"` + StatusType string `gorm:"type:varchar(100)"` + Type string `gorm:"type:varchar(100)"` + Priority string `gorm:"type:varchar(100)"` + Url string `gorm:"type:varchar(255)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedDate *time.Time + UpdatedDate *time.Time `gorm:"index"` + ClosedDate *time.Time + archived.NoPKModel +} + +func (ClickUpTask) TableName() string { return "_tool_clickup_tasks" } + +type ClickUpTaskComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TaskId string `gorm:"index;type:varchar(255)"` + Body string + UserId string `gorm:"type:varchar(255)"` + CreatedDate *time.Time + archived.NoPKModel +} + +func (ClickUpTaskComment) TableName() string { return "_tool_clickup_task_comments" } diff --git a/backend/plugins/clickup/models/migrationscripts/register.go b/backend/plugins/clickup/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/clickup/models/scope_config.go b/backend/plugins/clickup/models/scope_config.go new file mode 100644 index 00000000000..34988e26e0c --- /dev/null +++ b/backend/plugins/clickup/models/scope_config.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpScopeConfig allows a user to override how ClickUp raw statuses and +// task types map onto DevLake's standard domain values. +// +// Status: ClickUp statuses carry a `type` (open/unstarted/custom/done/closed) +// which the plugin maps automatically (open/unstarted -> TODO, custom -> +// IN_PROGRESS, done/closed -> DONE). When any of the IssueStatus* lists below +// are populated, a matching raw status name takes precedence over the +// type-derived default, so teams with bespoke workflows can classify custom +// statuses explicitly. +// +// Type: ClickUp has no native issue "type" on every task, so IssueType* are +// regular expressions matched against a task's derived type string. Precedence +// is INCIDENT > BUG > REQUIREMENT; a task matching none defaults to REQUIREMENT. +// A sensible default (bug -> BUG) is applied by the convertor when no config is +// set. +type ClickUpScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + IssueStatusTodo []string `mapstructure:"issueStatusTodo,omitempty" json:"issueStatusTodo" gorm:"type:json;serializer:json"` + IssueStatusInProgress []string `mapstructure:"issueStatusInProgress,omitempty" json:"issueStatusInProgress" gorm:"type:json;serializer:json"` + IssueStatusDone []string `mapstructure:"issueStatusDone,omitempty" json:"issueStatusDone" gorm:"type:json;serializer:json"` + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (ClickUpScopeConfig) TableName() string { + return "_tool_clickup_scope_configs" +} + +func (sc *ClickUpScopeConfig) SetConnectionId(c *ClickUpScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/clickup/models/task.go b/backend/plugins/clickup/models/task.go new file mode 100644 index 00000000000..60588e9ae99 --- /dev/null +++ b/backend/plugins/clickup/models/task.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpTask is the tool-layer representation of a ClickUp task. +type ClickUpTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + ListId string `gorm:"index;type:varchar(255)" json:"listId"` + SpaceId string `gorm:"type:varchar(255)" json:"spaceId"` + CustomId string `gorm:"type:varchar(255)" json:"customId"` + Name string `json:"name"` + Description string `json:"description"` + Status string `gorm:"type:varchar(255)" json:"status"` + StatusType string `gorm:"type:varchar(100)" json:"statusType"` + Type string `gorm:"type:varchar(100)" json:"type"` + Priority string `gorm:"type:varchar(100)" json:"priority"` + Url string `gorm:"type:varchar(255)" json:"url"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + AssigneeName string `gorm:"type:varchar(255)" json:"assigneeName"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedDate *time.Time `json:"createdDate"` + UpdatedDate *time.Time `gorm:"index" json:"updatedDate"` + ClosedDate *time.Time `json:"closedDate"` + common.NoPKModel +} + +func (ClickUpTask) TableName() string { + return "_tool_clickup_tasks" +} diff --git a/backend/plugins/clickup/models/task_comment.go b/backend/plugins/clickup/models/task_comment.go new file mode 100644 index 00000000000..c49d857d42c --- /dev/null +++ b/backend/plugins/clickup/models/task_comment.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpTaskComment is the tool-layer representation of a comment on a ClickUp +// task. +// +// TODO(clickup): the comment collector/extractor is not yet implemented. This +// model exists so the table is created up-front and the convertor to +// ticket.IssueComment can be added without a follow-up migration. See +// GET /task/{id}/comment. +type ClickUpTaskComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TaskId string `gorm:"index;type:varchar(255)" json:"taskId"` + Body string `json:"body"` + UserId string `gorm:"type:varchar(255)" json:"userId"` + CreatedDate *time.Time `json:"createdDate"` + common.NoPKModel +} + +func (ClickUpTaskComment) TableName() string { + return "_tool_clickup_task_comments" +} diff --git a/backend/plugins/clickup/models/user.go b/backend/plugins/clickup/models/user.go new file mode 100644 index 00000000000..05ca2aae134 --- /dev/null +++ b/backend/plugins/clickup/models/user.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClickUpUser is a ClickUp user (tool layer), converted to crossdomain.Account. +type ClickUpUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Username string `gorm:"type:varchar(255)" json:"username"` + Email string `gorm:"type:varchar(255)" json:"email"` + Color string `gorm:"type:varchar(50)" json:"color"` + ProfilePicture string `gorm:"type:varchar(255)" json:"profilePicture"` + common.NoPKModel +} + +func (ClickUpUser) TableName() string { + return "_tool_clickup_users" +} diff --git a/backend/plugins/clickup/tasks/api_client.go b/backend/plugins/clickup/tasks/api_client.go new file mode 100644 index 00000000000..85333cd184e --- /dev/null +++ b/backend/plugins/clickup/tasks/api_client.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// DefaultEndpoint is the ClickUp REST v2 base URL. Note the trailing slash: +// devlake joins it with relative UrlTemplates. +const DefaultEndpoint = "https://api.clickup.com/api/v2/" + +// CreateApiClient creates a new rate-limited async API client for ClickUp. +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.ClickUpConnection) (*api.ApiAsyncClient, errors.Error) { + if connection.Endpoint == "" { + connection.Endpoint = DefaultEndpoint + } + apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + asyncApiClient, err := api.CreateAsyncApiClient(taskCtx, apiClient, nil) + if err != nil { + return nil, err + } + + return asyncApiClient, nil +} diff --git a/backend/plugins/clickup/tasks/board_convertor.go b/backend/plugins/clickup/tasks/board_convertor.go new file mode 100644 index 00000000000..bb82a4e1b84 --- /dev/null +++ b/backend/plugins/clickup/tasks/board_convertor.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// RAW_LIST_TABLE labels the raw-data lineage for the list-scope-derived board. +// Lists are added as scopes (no collector), so this is a logical tag only. +const RAW_LIST_TABLE = "clickup_lists" + +var ConvertListMeta = plugin.SubTaskMeta{ + Name: "Convert Lists", + EntryPoint: ConvertLists, + EnabledByDefault: true, + Description: "Convert the ClickUp list scope (_tool_clickup_lists) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpList{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertLists + +func ConvertLists(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the task convertor so the board + // joins to the board_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + + cursor, err := db.Cursor( + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND list_id = ?", connectionId, data.Options.ListId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + ListId: data.Options.ListId, + }, + Table: RAW_LIST_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpList{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + list := inputRow.(*models.ClickUpList) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, list.ListId)}, + Name: list.ScopeFullName(), + Type: "clickup", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/clickup/tasks/shared.go b/backend/plugins/clickup/tasks/shared.go new file mode 100644 index 00000000000..64d41df5c67 --- /dev/null +++ b/backend/plugins/clickup/tasks/shared.go @@ -0,0 +1,172 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "regexp" + "strconv" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// parseClickUpTime parses a ClickUp millisecond-epoch timestamp. ClickUp encodes +// timestamps as strings of milliseconds since the Unix epoch (e.g. "1567780450202"). +// It returns nil for empty / zero / unparseable values so callers can leave the +// corresponding *time.Time unset rather than storing a bogus 1970 date. +func parseClickUpTime(ms string) *time.Time { + ms = strings.TrimSpace(ms) + if ms == "" { + return nil + } + millis, err := strconv.ParseInt(ms, 10, 64) + if err != nil || millis <= 0 { + return nil + } + t := time.UnixMilli(millis).UTC() + return &t +} + +// statusFromType maps a ClickUp status.type onto a DevLake standard issue +// status. ClickUp's status types are standardized: +// +// open, unstarted -> TODO +// custom -> IN_PROGRESS +// done, closed -> DONE +// +// Any unrecognized type falls back to OTHER so unexpected API values surface +// rather than silently masquerading as a known status. +func statusFromType(statusType string) string { + switch strings.ToLower(statusType) { + case "open", "unstarted": + return ticket.TODO + case "custom": + return ticket.IN_PROGRESS + case "done", "closed": + return ticket.DONE + default: + return ticket.OTHER + } +} + +// statusMapper resolves a ClickUp task's domain status. When the scope config +// supplies explicit status name lists, a matching raw status name wins; +// otherwise the status.type-derived default is used. +type statusMapper struct { + byName map[string]string + hasList bool +} + +func newStatusMapper(sc *models.ClickUpScopeConfig) *statusMapper { + m := &statusMapper{byName: map[string]string{}} + if sc == nil { + return m + } + add := func(names []string, domain string) { + for _, n := range names { + n = strings.ToLower(strings.TrimSpace(n)) + if n != "" { + m.byName[n] = domain + m.hasList = true + } + } + } + add(sc.IssueStatusTodo, ticket.TODO) + add(sc.IssueStatusInProgress, ticket.IN_PROGRESS) + add(sc.IssueStatusDone, ticket.DONE) + return m +} + +// statusOf returns the domain status for a raw status name / type. A user-configured +// name mapping takes precedence over the type-derived default. +func (m *statusMapper) statusOf(rawStatus, statusType string) string { + if m.hasList { + if domain, ok := m.byName[strings.ToLower(strings.TrimSpace(rawStatus))]; ok { + return domain + } + } + return statusFromType(statusType) +} + +// issueTypeMatcher derives the domain ticket.Issue.Type from a task's derived +// type string using the scope config's regex patterns. Precedence is +// INCIDENT > BUG > REQUIREMENT; a task matching none defaults to REQUIREMENT. +// +// When no patterns are configured a sensible default (bug -> BUG) is applied so +// bug-typed tasks feed DORA change-failure-rate out of the box. +type issueTypeMatcher struct { + incident *regexp.Regexp + bug *regexp.Regexp + requirement *regexp.Regexp +} + +// defaultBugPattern matches ClickUp's built-in "Bug" custom task type name +// (case-insensitive) when the scope config leaves IssueTypeBug empty. +const defaultBugPattern = "(?i)^bug$" + +func newIssueTypeMatcher(sc *models.ClickUpScopeConfig) (*issueTypeMatcher, errors.Error) { + m := &issueTypeMatcher{} + bugPattern := defaultBugPattern + var incidentPattern, requirementPattern string + if sc != nil { + if sc.IssueTypeBug != "" { + bugPattern = sc.IssueTypeBug + } + incidentPattern = sc.IssueTypeIncident + requirementPattern = sc.IssueTypeRequirement + } + for _, p := range []struct { + pattern string + field string + out **regexp.Regexp + }{ + {incidentPattern, "issueTypeIncident", &m.incident}, + {bugPattern, "issueTypeBug", &m.bug}, + {requirementPattern, "issueTypeRequirement", &m.requirement}, + } { + if p.pattern == "" { + continue + } + re, err := errors.Convert01(regexp.Compile(p.pattern)) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid "+p.field+" pattern") + } + *p.out = re + } + return m, nil +} + +// typeOf returns the domain issue type for a task's derived type string. +func (m *issueTypeMatcher) typeOf(taskType string) string { + for _, c := range []struct { + pattern *regexp.Regexp + typ string + }{ + {m.incident, ticket.INCIDENT}, + {m.bug, ticket.BUG}, + {m.requirement, ticket.REQUIREMENT}, + } { + if c.pattern != nil && c.pattern.MatchString(taskType) { + return c.typ + } + } + return ticket.REQUIREMENT +} diff --git a/backend/plugins/clickup/tasks/task_collector.go b/backend/plugins/clickup/tasks/task_collector.go new file mode 100644 index 00000000000..afc92d99e76 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_collector.go @@ -0,0 +1,111 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_TASK_TABLE = "clickup_tasks" + +// clickUpTaskListResponse mirrors the envelope returned by +// GET /list/{id}/task. ClickUp uses zero-based page numbers and signals the end +// of the collection with `last_page: true` (and/or an empty `tasks` array). +type clickUpTaskListResponse struct { + Tasks []json.RawMessage `json:"tasks"` + LastPage bool `json:"last_page"` +} + +var CollectTaskMeta = plugin.SubTaskMeta{ + Name: "Collect Tasks", + EntryPoint: CollectTasks, + EnabledByDefault: true, + Description: "Collect tasks for a ClickUp list (page-based pagination)", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectTasks + +func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + ListId: data.Options.ListId, + }, + Table: RAW_TASK_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 100, + UrlTemplate: "list/{{ .Params.ListId }}/task", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("subtasks", "true") + query.Set("include_closed", "true") + page := "0" + if reqData.CustomData != nil { + if p, ok := reqData.CustomData.(string); ok && p != "" { + page = p + } + } + query.Set("page", page) + // Incremental collection: restrict to tasks updated after the + // configured cut-off (ClickUp expects milliseconds since epoch). + if data.TimeAfter != nil { + query.Set("date_updated_gt", strconv.FormatInt(data.TimeAfter.UnixMilli(), 10)) + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp clickUpTaskListResponse + if err := api.UnmarshalResponse(prevPageResponse, &resp); err != nil { + return nil, err + } + if resp.LastPage || len(resp.Tasks) == 0 { + return nil, api.ErrFinishCollect + } + prevPage := 0 + if prevReqData.CustomData != nil { + if p, ok := prevReqData.CustomData.(string); ok { + prevPage, _ = strconv.Atoi(p) + } + } + return strconv.Itoa(prevPage + 1), nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpTaskListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Tasks, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/task_convertor.go b/backend/plugins/clickup/tasks/task_convertor.go new file mode 100644 index 00000000000..b779019144c --- /dev/null +++ b/backend/plugins/clickup/tasks/task_convertor.go @@ -0,0 +1,143 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ConvertTaskMeta = plugin.SubTaskMeta{ + Name: "Convert Tasks", + EntryPoint: ConvertTasks, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_clickup_tasks into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpTask{}.TableName(), RAW_TASK_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTasks + +func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.ClickUpTask{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.ClickUpUser{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + boardId := boardIdGen.Generate(connectionId, data.Options.ListId) + + statusMapper := newStatusMapper(data.ScopeConfig) + typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) + if err != nil { + return err + } + + cursor, err := db.Cursor( + dal.From(&models.ClickUpTask{}), + dal.Where("connection_id = ? AND list_id = ?", connectionId, data.Options.ListId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + ListId: data.Options.ListId, + }, + Table: RAW_TASK_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpTask{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + task := inputRow.(*models.ClickUpTask) + + issueKey := task.CustomId + if issueKey == "" { + issueKey = task.Id + } + + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, task.Id)}, + IssueKey: issueKey, + Title: task.Name, + Description: task.Description, + Url: task.Url, + Type: typeMatcher.typeOf(task.Type), + OriginalType: task.Type, + Status: statusMapper.statusOf(task.Status, task.StatusType), + OriginalStatus: task.Status, + Priority: task.Priority, + CreatedDate: task.CreatedDate, + UpdatedDate: task.UpdatedDate, + ResolutionDate: task.ClosedDate, + } + if task.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, task.CreatorId) + } + if task.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, task.AssigneeId) + domainIssue.AssigneeName = task.AssigneeName + } + if task.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, task.ParentId) + domainIssue.IsSubtask = true + } + // Fallback lead time. Guard against a resolution that precedes + // creation (clock skew / imported tasks): a negative duration cast + // to uint yields garbage, so leave lead time unset instead. + if domainIssue.ResolutionDate != nil && task.CreatedDate != nil && + domainIssue.ResolutionDate.After(*task.CreatedDate) { + minutes := uint(domainIssue.ResolutionDate.Sub(*task.CreatedDate).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/clickup/tasks/task_data.go b/backend/plugins/clickup/tasks/task_data.go new file mode 100644 index 00000000000..8b6bd0f9719 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_data.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// ClickUpOptions are the per-scope options passed to a pipeline task. +type ClickUpOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + ListId string `json:"listId" mapstructure:"listId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// ClickUpTaskData is the shared context handed to every ClickUp subtask. +type ClickUpTaskData struct { + Options *ClickUpOptions + ApiClient *api.ApiAsyncClient + TimeAfter *time.Time + // ScopeConfig carries the resolved scope config (status + type mapping). + // Never nil: PrepareTaskData defaults it to an empty config. + ScopeConfig *models.ClickUpScopeConfig +} + +type ClickUpApiParams models.ClickUpApiParams diff --git a/backend/plugins/clickup/tasks/task_extractor.go b/backend/plugins/clickup/tasks/task_extractor.go new file mode 100644 index 00000000000..5bee7ecff28 --- /dev/null +++ b/backend/plugins/clickup/tasks/task_extractor.go @@ -0,0 +1,152 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractTaskMeta = plugin.SubTaskMeta{ + Name: "Extract Tasks", + EntryPoint: ExtractTasks, + EnabledByDefault: true, + Description: "Extract raw task data into the tool layer table _tool_clickup_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractTasks + +// ClickUpApiTask is the subset of the ClickUp task JSON that the extractor reads. +type ClickUpApiTask struct { + Id string `json:"id"` + CustomId string `json:"custom_id"` + Name string `json:"name"` + TextContent string `json:"text_content"` + Description string `json:"markdown_description"` + Status *struct { + Status string `json:"status"` + Type string `json:"type"` + } `json:"status"` + DateCreated string `json:"date_created"` + DateUpdated string `json:"date_updated"` + DateClosed string `json:"date_closed"` + Creator *struct { + Id json.Number `json:"id"` + Username string `json:"username"` + } `json:"creator"` + Assignees []struct { + Id json.Number `json:"id"` + Username string `json:"username"` + } `json:"assignees"` + Priority *struct { + Priority string `json:"priority"` + } `json:"priority"` + Parent string `json:"parent"` + Url string `json:"url"` + TaskType string `json:"task_type"` + List *struct { + Id string `json:"id"` + } `json:"list"` + Space *struct { + Id string `json:"id"` + } `json:"space"` +} + +func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + ListId: data.Options.ListId, + }, + Table: RAW_TASK_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiTask := &ClickUpApiTask{} + if err := errors.Convert(json.Unmarshal(row.Data, apiTask)); err != nil { + return nil, err + } + if apiTask.Id == "" { + return nil, nil + } + description := apiTask.Description + if description == "" { + description = apiTask.TextContent + } + listId := data.Options.ListId + if apiTask.List != nil && apiTask.List.Id != "" { + listId = apiTask.List.Id + } + task := &models.ClickUpTask{ + ConnectionId: data.Options.ConnectionId, + Id: apiTask.Id, + ListId: listId, + CustomId: apiTask.CustomId, + Name: apiTask.Name, + Description: description, + Type: apiTask.TaskType, + ParentId: parentOf(apiTask.Parent), + Url: apiTask.Url, + CreatedDate: parseClickUpTime(apiTask.DateCreated), + UpdatedDate: parseClickUpTime(apiTask.DateUpdated), + ClosedDate: parseClickUpTime(apiTask.DateClosed), + } + if apiTask.Space != nil { + task.SpaceId = apiTask.Space.Id + } + if apiTask.Status != nil { + task.Status = apiTask.Status.Status + task.StatusType = apiTask.Status.Type + } + if apiTask.Priority != nil { + task.Priority = apiTask.Priority.Priority + } + if apiTask.Creator != nil { + task.CreatorId = apiTask.Creator.Id.String() + } + // TODO(clickup): ClickUp tasks can have multiple assignees. The MVP + // keeps only the first for Issue.AssigneeId; emitting one + // ticket.IssueAssignee row per assignee is a follow-up. + if len(apiTask.Assignees) > 0 { + task.AssigneeId = apiTask.Assignees[0].Id.String() + task.AssigneeName = apiTask.Assignees[0].Username + } + return []interface{}{task}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +// parentOf normalizes ClickUp's parent field, which is the JSON literal null +// (decoded to an empty string) for top-level tasks. +func parentOf(parent string) string { + if parent == "null" { + return "" + } + return parent +} diff --git a/backend/plugins/clickup/tasks/user_collector.go b/backend/plugins/clickup/tasks/user_collector.go new file mode 100644 index 00000000000..421b598f95c --- /dev/null +++ b/backend/plugins/clickup/tasks/user_collector.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_USER_TABLE = "clickup_users" + +// clickUpMemberListResponse mirrors the envelope returned by +// GET /list/{id}/member. +type clickUpMemberListResponse struct { + Members []json.RawMessage `json:"members"` +} + +var CollectUserMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectUsers, + EnabledByDefault: true, + Description: "Collect the members of a ClickUp list", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectUsers + +func CollectUsers(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + ListId: data.Options.ListId, + }, + Table: RAW_USER_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "list/{{ .Params.ListId }}/member", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpMemberListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Members, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/user_convertor.go b/backend/plugins/clickup/tasks/user_convertor.go new file mode 100644 index 00000000000..1ff4f0c3d6d --- /dev/null +++ b/backend/plugins/clickup/tasks/user_convertor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ConvertUserMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertUsers, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_clickup_users into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.ClickUpUser{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertUsers + +func ConvertUsers(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.ClickUpUser{}) + + cursor, err := db.Cursor( + dal.From(&models.ClickUpUser{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + ListId: data.Options.ListId, + }, + Table: RAW_USER_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpUser{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + user := inputRow.(*models.ClickUpUser) + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, user.Id), + }, + UserName: user.Username, + FullName: user.Username, + Email: user.Email, + AvatarUrl: user.ProfilePicture, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/clickup/tasks/user_extractor.go b/backend/plugins/clickup/tasks/user_extractor.go new file mode 100644 index 00000000000..d35aa811d66 --- /dev/null +++ b/backend/plugins/clickup/tasks/user_extractor.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractUserMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractUsers, + EnabledByDefault: true, + Description: "Extract raw member data into the tool layer table _tool_clickup_users", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractUsers + +// ClickUpApiUser is the subset of a ClickUp member JSON that the extractor reads. +type ClickUpApiUser struct { + Id json.Number `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + Color string `json:"color"` + ProfilePicture string `json:"profilePicture"` +} + +func ExtractUsers(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + ListId: data.Options.ListId, + }, + Table: RAW_USER_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiUser := &ClickUpApiUser{} + if err := errors.Convert(json.Unmarshal(row.Data, apiUser)); err != nil { + return nil, err + } + id := apiUser.Id.String() + if id == "" { + return nil, nil + } + user := &models.ClickUpUser{ + ConnectionId: data.Options.ConnectionId, + Id: id, + Username: apiUser.Username, + Email: apiUser.Email, + Color: apiUser.Color, + ProfilePicture: apiUser.ProfilePicture, + } + return []interface{}{user}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 0d4482ca6c9..2f2ba5e0ae4 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -30,6 +30,7 @@ import ( bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" + clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" @@ -107,6 +108,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("zentao/models", zentao.Zentao{}.GetTablesInfo) checker.FeedIn("claude_code/models", claudeCode.ClaudeCode{}.GetTablesInfo) checker.FeedIn("circleci/models", circleci.Circleci{}.GetTablesInfo) + checker.FeedIn("clickup/models", clickup.ClickUp{}.GetTablesInfo) checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) From 993527383946ebd5c188273c2107b77808bcf599 Mon Sep 17 00:00:00 2001 From: Evan Lynch Date: Tue, 21 Jul 2026 17:35:05 -0400 Subject: [PATCH 2/3] feat(clickup): add ClickUp connector with folder-scope boards, sprints, and DORA Add a new ClickUp data-source plugin so ClickUp workspaces feed DevLake's issue-tracking and DORA/velocity metrics, modeled on the Jira/Linear connectors. Backend: - Folder is the data-source scope (ClickUpFolder ~= a Jira board); lists are child entities collected every sync, so rolling/archived sprint lists never need re-scoping. - Sprint lists (name-matched) -> ticket.Sprint + board_sprints; start/end parsed from the list name (M/D vs D/M disambiguated by the >12 rule). - Tasks -> issues + board_issues; sprint-list tasks also -> sprint_issues. - Story points from ClickUp native sprint points (or a configured custom field). - Scope config: sprint-name pattern, story-point field, force-issue-type (flag a folder INCIDENT for DORA CFR/MTTR), status/type overrides. - remote-scopes walks Workspace -> Space -> Folder (folders selectable). - migration 20260722 adds _tool_clickup_folders + sprint/story-point columns. - Unit tests for sprint name/date parsing and status/type/story-point mapping. Config UI: - ClickUp connection registration (personal-token auth) with a Beta pill. - Boards data-scope picker + scope-config transformation panel. - getPluginScopeId(clickup) -> folderId so scope delete and per-scope project selection target one scope; fix a pre-existing Connections-page chunk() split that silently dropped plugins. Docs & dashboards: - Plugin README (connection, folder scope, sprints, story points, incidents). - Branded ClickUp Grafana dashboard (grafana/dashboards/ClickUp.json). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Evan Lynch --- backend/plugins/clickup/README.md | 92 ++ backend/plugins/clickup/api/blueprint_v200.go | 10 +- backend/plugins/clickup/api/init.go | 8 +- backend/plugins/clickup/api/remote_api.go | 105 +- backend/plugins/clickup/api/scope_api.go | 10 +- backend/plugins/clickup/impl/impl.go | 17 +- backend/plugins/clickup/models/folder.go | 72 + backend/plugins/clickup/models/list.go | 62 +- .../20260722_add_folder_scope.go | 102 ++ .../models/migrationscripts/register.go | 1 + .../plugins/clickup/models/scope_config.go | 14 + backend/plugins/clickup/models/task.go | 2 + ...board_convertor.go => folder_convertor.go} | 43 +- .../plugins/clickup/tasks/list_collector.go | 111 ++ .../plugins/clickup/tasks/list_extractor.go | 102 ++ backend/plugins/clickup/tasks/shared.go | 82 ++ backend/plugins/clickup/tasks/shared_test.go | 254 ++++ .../plugins/clickup/tasks/sprint_convertor.go | 124 ++ .../plugins/clickup/tasks/task_collector.go | 38 +- .../plugins/clickup/tasks/task_convertor.go | 67 +- backend/plugins/clickup/tasks/task_data.go | 2 +- .../plugins/clickup/tasks/task_extractor.go | 58 +- .../plugins/clickup/tasks/user_collector.go | 4 +- .../plugins/clickup/tasks/user_convertor.go | 2 +- .../plugins/clickup/tasks/user_extractor.go | 2 +- .../components/scope-config-form/index.tsx | 10 + .../plugins/register/clickup/assets/icon.svg | 20 + .../src/plugins/register/clickup/config.tsx | 79 ++ .../src/plugins/register/clickup/index.ts | 20 + .../register/clickup/transformation.tsx | 160 +++ config-ui/src/plugins/register/index.ts | 2 + config-ui/src/plugins/utils.ts | 2 + .../src/routes/connection/connections.tsx | 7 +- grafana/dashboards/mysql/ClickUp.json | 1252 +++++++++++++++++ grafana/dashboards/postgresql/ClickUp.json | 1252 +++++++++++++++++ 35 files changed, 4013 insertions(+), 175 deletions(-) create mode 100644 backend/plugins/clickup/README.md create mode 100644 backend/plugins/clickup/models/folder.go create mode 100644 backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go rename backend/plugins/clickup/tasks/{board_convertor.go => folder_convertor.go} (64%) create mode 100644 backend/plugins/clickup/tasks/list_collector.go create mode 100644 backend/plugins/clickup/tasks/list_extractor.go create mode 100644 backend/plugins/clickup/tasks/shared_test.go create mode 100644 backend/plugins/clickup/tasks/sprint_convertor.go create mode 100644 config-ui/src/plugins/register/clickup/assets/icon.svg create mode 100644 config-ui/src/plugins/register/clickup/config.tsx create mode 100644 config-ui/src/plugins/register/clickup/index.ts create mode 100644 config-ui/src/plugins/register/clickup/transformation.tsx create mode 100644 grafana/dashboards/mysql/ClickUp.json create mode 100644 grafana/dashboards/postgresql/ClickUp.json diff --git a/backend/plugins/clickup/README.md b/backend/plugins/clickup/README.md new file mode 100644 index 00000000000..1d4122a5bbb --- /dev/null +++ b/backend/plugins/clickup/README.md @@ -0,0 +1,92 @@ + + +# ClickUp + +The ClickUp plugin collects issues, boards, and sprints from ClickUp so they +feed DevLake's issue-tracking and DORA/velocity metrics, modeled on the Jira +and Linear connectors. + +## Authentication + +ClickUp authenticates with a **personal API token** (ClickUp → Settings → Apps +→ API Token, starts with `pk_`). The token is sent verbatim in the +`Authorization` header. OAuth is not yet supported. + +Create a connection with: + +- **Endpoint** — `https://api.clickup.com/api/v2/` +- **Token** — your `pk_...` personal token + +## Data scope: the folder is the board + +Unlike a raw list, the **scope you select is a ClickUp folder** (e.g. a team's +`Dev Team` / `Sprint Folder`). This mirrors a Jira board: selecting the folder +collects every list inside it on each sync, so rolling and archived sprint +lists never need to be re-scoped. + +Domain mapping: + +| ClickUp | DevLake domain | +| --- | --- | +| Folder | `board` | +| Sprint list (name-matched) | `sprint` + `board_sprint` | +| Task | `issue` + `board_issue` | +| Task in a sprint list | `sprint_issue` | +| Folder member | `account` | + +## Sprints + +A list is treated as a sprint when its name matches the **sprint name pattern** +(default `(?i)sprint\s*\d+`, e.g. `v4.3.0 Sprint 40 (7/6/26 - 7/19/26)`). The +start/end dates are parsed from the parenthesised date span in the name; +`M/D/YY` vs `D/M/YY` ordering is disambiguated automatically (a component > 12 +must be the day). Lists that don't match are collected as plain board issues. +Archived sprint lists are collected too, so historical velocity is retained. + +## Story points + +Story points default to ClickUp's native sprint **`points`** field. To read +them from a custom field instead (e.g. a Fibonacci "LOE" field), set +**Story point field** in the scope config to the custom-field name. + +## Incidents (DORA Change Failure Rate / MTTR) + +ClickUp tasks have no universal "type" field, so incidents are modeled by +**scope**: add the folder that holds incidents (e.g. a "Security Incidents" +folder) as its own board and set the scope config's **Force issue type** to +`INCIDENT`. Every issue on that board is then classified as an incident. + +## Scope configuration (transformation) + +Per board, the scope config lets you override: + +- **Sprint name pattern** — which lists are sprints. +- **Story point field** — native `points` (blank) or a custom field name. +- **Force issue type** — flag the whole board as `REQUIREMENT` / `BUG` / + `INCIDENT` (leave blank to detect per task). +- **Issue type patterns** — RegExes matched against a task's type; precedence + is `INCIDENT` > `BUG` > `REQUIREMENT`. +- **Status mapping** — ClickUp statuses are auto-mapped by their `type` + (`open`/`unstarted` → `TODO`, `custom` → `IN_PROGRESS`, `done`/`closed` → + `DONE`); list raw status names to override where a team's workflow differs. + +## Metrics + +A branded **ClickUp** Grafana dashboard (`grafana/dashboards/{mysql,postgresql}/ClickUp.json`) +renders issue throughput and lead-time metrics; DORA and sprint/velocity +metrics are computed from the source-agnostic domain layer. diff --git a/backend/plugins/clickup/api/blueprint_v200.go b/backend/plugins/clickup/api/blueprint_v200.go index b2c02162bee..98f2439d696 100644 --- a/backend/plugins/clickup/api/blueprint_v200.go +++ b/backend/plugins/clickup/api/blueprint_v200.go @@ -53,7 +53,7 @@ func MakePipelinePlanV200( func makePipelinePlanV200( subtaskMetas []plugin.SubTaskMeta, - scopeDetails []*srvhelper.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig], connection *models.ClickUpConnection, ) (coreModels.PipelinePlan, errors.Error) { plan := make(coreModels.PipelinePlan, len(scopeDetails)) @@ -69,7 +69,7 @@ func makePipelinePlanV200( scopeConfig.Entities, tasks.ClickUpOptions{ ConnectionId: connection.ID, - ListId: scope.ListId, + FolderId: scope.FolderId, ScopeConfigId: scope.ScopeConfigId, }, ) @@ -83,14 +83,14 @@ func makePipelinePlanV200( } func makeScopesV200( - scopeDetails []*srvhelper.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig], connection *models.ClickUpConnection, ) ([]plugin.Scope, errors.Error) { scopes := make([]plugin.Scope, 0, len(scopeDetails)) - idgen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + idgen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) for _, scopeDetail := range scopeDetails { scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig - id := idgen.Generate(connection.ID, scope.ListId) + id := idgen.Generate(connection.ID, scope.FolderId) if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { scopes = append(scopes, ticket.NewBoard(id, scope.Name)) } diff --git a/backend/plugins/clickup/api/init.go b/backend/plugins/clickup/api/init.go index 0f484165433..bc185fbd5b0 100644 --- a/backend/plugins/clickup/api/init.go +++ b/backend/plugins/clickup/api/init.go @@ -27,15 +27,15 @@ import ( var vld *validator.Validate var basicRes context.BasicRes -var dsHelper *api.DsHelper[models.ClickUpConnection, models.ClickUpList, models.ClickUpScopeConfig] +var dsHelper *api.DsHelper[models.ClickUpConnection, models.ClickUpFolder, models.ClickUpScopeConfig] var raProxy *api.DsRemoteApiProxyHelper[models.ClickUpConnection] -var raScopeList *api.DsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpList, ClickUpRemotePagination] +var raScopeList *api.DsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpFolder, ClickUpRemotePagination] func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br vld = validator.New() dsHelper = api.NewDataSourceHelper[ - models.ClickUpConnection, models.ClickUpList, models.ClickUpScopeConfig, + models.ClickUpConnection, models.ClickUpFolder, models.ClickUpScopeConfig, ]( br, p.Name(), @@ -47,5 +47,5 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { nil, ) raProxy = api.NewDsRemoteApiProxyHelper[models.ClickUpConnection](dsHelper.ConnApi.ModelApiHelper) - raScopeList = api.NewDsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpList, ClickUpRemotePagination](raProxy, listClickUpRemoteScopes) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.ClickUpConnection, models.ClickUpFolder, ClickUpRemotePagination](raProxy, listClickUpRemoteScopes) } diff --git a/backend/plugins/clickup/api/remote_api.go b/backend/plugins/clickup/api/remote_api.go index 5734020d30b..93e9589c93b 100644 --- a/backend/plugins/clickup/api/remote_api.go +++ b/backend/plugins/clickup/api/remote_api.go @@ -29,23 +29,23 @@ import ( ) // ClickUpRemotePagination is a placeholder: the ClickUp v2 hierarchy endpoints -// used here (team/space/folder/list) are not paginated, so there is never a -// next page. It exists to satisfy the DsRemoteApiScopeListHelper generic. +// used here (team/space/folder) are not paginated, so there is never a next +// page. It exists to satisfy the DsRemoteApiScopeListHelper generic. type ClickUpRemotePagination struct{} // groupId prefixes encode which level of the ClickUp hierarchy a group entry -// points at, so a single list function can walk Team -> Space -> Folder -> List. +// points at, so a single list function can walk Team -> Space -> Folder. The +// selectable scope is the Folder (= board); spaces and teams are navigation +// groups only. const ( - groupTeamPrefix = "team:" - groupSpacePrefix = "space:" - groupFolderPrefix = "folder:" + groupTeamPrefix = "team:" + groupSpacePrefix = "space:" ) type clickUpNamedEntities struct { Teams []clickUpNamedEntity `json:"teams"` Spaces []clickUpNamedEntity `json:"spaces"` Folders []clickUpNamedEntity `json:"folders"` - Lists []clickUpNamedEntity `json:"lists"` } type clickUpNamedEntity struct { @@ -58,15 +58,14 @@ type clickUpNamedEntity struct { // // "" -> workspaces (Team) as groups // team:{id} -> spaces as groups -// space:{id} -> folders as groups + folderless lists as selectable scopes -// folder:{id} -> lists as selectable scopes +// space:{id} -> folders as selectable scopes (boards) func listClickUpRemoteScopes( _ *models.ClickUpConnection, apiClient plugin.ApiClient, groupId string, _ ClickUpRemotePagination, ) ( - children []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], + children []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], nextPage *ClickUpRemotePagination, err errors.Error, ) { @@ -74,11 +73,9 @@ func listClickUpRemoteScopes( case groupId == "": return listTeamsAsGroups(apiClient) case strings.HasPrefix(groupId, groupTeamPrefix): - return listSpacesAsGroups(apiClient, strings.TrimPrefix(groupId, groupTeamPrefix)) + return listSpacesAsGroups(apiClient, strings.TrimPrefix(groupId, groupTeamPrefix), groupId) case strings.HasPrefix(groupId, groupSpacePrefix): - return listSpaceChildren(apiClient, strings.TrimPrefix(groupId, groupSpacePrefix)) - case strings.HasPrefix(groupId, groupFolderPrefix): - return listFolderLists(apiClient, strings.TrimPrefix(groupId, groupFolderPrefix)) + return listFoldersAsScopes(apiClient, strings.TrimPrefix(groupId, groupSpacePrefix), groupId) default: return nil, nil, errors.BadInput.New(fmt.Sprintf("unrecognized groupId %q", groupId)) } @@ -96,88 +93,72 @@ func getEntities(apiClient plugin.ApiClient, path string) (*clickUpNamedEntities return &body, nil } -func groupEntry(id, name string) dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList] { - return dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList]{ +// groupEntry builds a navigation-group row. parentId is the id of the group +// this row is nested under (nil for the top level); the miller-column UI uses +// it to render children in the next column instead of inline. +func groupEntry(id, name string, parentId *string) dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder] { + return dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder]{ Type: api.RAS_ENTRY_TYPE_GROUP, - ParentId: nil, + ParentId: parentId, Id: id, Name: name, FullName: name, } } -func listTeamsAsGroups(apiClient plugin.ApiClient) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { +func listTeamsAsGroups(apiClient plugin.ApiClient) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { body, err := getEntities(apiClient, "team") if err != nil { return nil, nil, err } - children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(body.Teams)) + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Teams)) for _, team := range body.Teams { - children = append(children, groupEntry(groupTeamPrefix+team.Id, team.Name)) + children = append(children, groupEntry(groupTeamPrefix+team.Id, team.Name, nil)) } return children, nil, nil } -func listSpacesAsGroups(apiClient plugin.ApiClient, teamId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { +func listSpacesAsGroups(apiClient plugin.ApiClient, teamId, parentId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { body, err := getEntities(apiClient, fmt.Sprintf("team/%s/space", teamId)) if err != nil { return nil, nil, err } - children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(body.Spaces)) + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Spaces)) for _, space := range body.Spaces { - children = append(children, groupEntry(groupSpacePrefix+space.Id, space.Name)) + children = append(children, groupEntry(groupSpacePrefix+space.Id, space.Name, &parentId)) } return children, nil, nil } -func listSpaceChildren(apiClient plugin.ApiClient, spaceId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { - folders, err := getEntities(apiClient, fmt.Sprintf("space/%s/folder", spaceId)) +// listFoldersAsScopes returns a space's folders as selectable (leaf) scope +// entries — the folder is the board a user picks. parentId nests them under the +// space column. +func listFoldersAsScopes(apiClient plugin.ApiClient, spaceId, parentId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], *ClickUpRemotePagination, errors.Error) { + body, err := getEntities(apiClient, fmt.Sprintf("space/%s/folder", spaceId)) if err != nil { return nil, nil, err } - children := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0) - for _, folder := range folders.Folders { - children = append(children, groupEntry(groupFolderPrefix+folder.Id, folder.Name)) - } - // Folderless lists live directly under the space. - lists, err := getEntities(apiClient, fmt.Sprintf("space/%s/list", spaceId)) - if err != nil { - return nil, nil, err - } - children = append(children, listsToScopeEntries(lists.Lists, spaceId)...) - return children, nil, nil -} - -func listFolderLists(apiClient plugin.ApiClient, folderId string) ([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], *ClickUpRemotePagination, errors.Error) { - body, err := getEntities(apiClient, fmt.Sprintf("folder/%s/list", folderId)) - if err != nil { - return nil, nil, err - } - return listsToScopeEntries(body.Lists, ""), nil, nil -} - -// listsToScopeEntries maps ClickUp lists into selectable (leaf) scope entries. -func listsToScopeEntries(lists []clickUpNamedEntity, spaceId string) []dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList] { - entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList], 0, len(lists)) - for _, list := range lists { - list := list - entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.ClickUpList]{ + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder], 0, len(body.Folders)) + for _, folder := range body.Folders { + folder := folder + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.ClickUpFolder]{ Type: api.RAS_ENTRY_TYPE_SCOPE, - ParentId: nil, - Id: list.Id, - Name: list.Name, - FullName: list.Name, - Data: &models.ClickUpList{ - ListId: list.Id, - Name: list.Name, - SpaceId: spaceId, + ParentId: &parentId, + Id: folder.Id, + Name: folder.Name, + FullName: folder.Name, + Data: &models.ClickUpFolder{ + FolderId: folder.Id, + Name: folder.Name, + SpaceId: spaceId, + SpaceName: "", }, }) } - return entries + return entries, nil, nil } -// RemoteScopes lists the ClickUp lists available on the connection so the +// RemoteScopes lists the ClickUp folders available on the connection so the // config UI can enumerate selectable scopes. func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return raScopeList.Get(input) diff --git a/backend/plugins/clickup/api/scope_api.go b/backend/plugins/clickup/api/scope_api.go index 5cc7d012ad8..5bfcb9d5aef 100644 --- a/backend/plugins/clickup/api/scope_api.go +++ b/backend/plugins/clickup/api/scope_api.go @@ -24,8 +24,8 @@ import ( "github.com/apache/incubator-devlake/plugins/clickup/models" ) -type PutScopesReqBody api.PutScopesReqBody[models.ClickUpList] -type ScopeDetail api.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig] +type PutScopesReqBody api.PutScopesReqBody[models.ClickUpFolder] +type ScopeDetail api.ScopeDetail[models.ClickUpFolder, models.ClickUpScopeConfig] // PutScopes create or update clickup lists // @Summary create or update clickup lists @@ -34,7 +34,7 @@ type ScopeDetail api.ScopeDetail[models.ClickUpList, models.ClickUpScopeConfig] // @Accept application/json // @Param connectionId path int false "connection ID" // @Param scope body PutScopesReqBody true "json" -// @Success 200 {object} []models.ClickUpList +// @Success 200 {object} []models.ClickUpFolder // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/clickup/connections/{connectionId}/scopes [PUT] @@ -49,8 +49,8 @@ func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, error // @Accept application/json // @Param connectionId path int false "connection ID" // @Param scopeId path string false "list ID" -// @Param scope body models.ClickUpList true "json" -// @Success 200 {object} models.ClickUpList +// @Param scope body models.ClickUpFolder true "json" +// @Success 200 {object} models.ClickUpFolder // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/clickup/connections/{connectionId}/scopes/{scopeId} [PATCH] diff --git a/backend/plugins/clickup/impl/impl.go b/backend/plugins/clickup/impl/impl.go index d578392f90b..bdf9b6e7e42 100644 --- a/backend/plugins/clickup/impl/impl.go +++ b/backend/plugins/clickup/impl/impl.go @@ -69,7 +69,7 @@ func (p ClickUp) Connection() dal.Tabler { } func (p ClickUp) Scope() plugin.ToolLayerScope { - return &models.ClickUpList{} + return &models.ClickUpFolder{} } func (p ClickUp) ScopeConfig() dal.Tabler { @@ -84,6 +84,7 @@ func (p ClickUp) MigrationScripts() []plugin.MigrationScript { func (p ClickUp) GetTablesInfo() []dal.Tabler { return []dal.Tabler{ &models.ClickUpConnection{}, + &models.ClickUpFolder{}, &models.ClickUpList{}, &models.ClickUpScopeConfig{}, &models.ClickUpUser{}, @@ -93,15 +94,19 @@ func (p ClickUp) GetTablesInfo() []dal.Tabler { } // SubTaskMetas lists subtasks in dependency order: collect/extract before -// convert; users before tasks (issues reference accounts); lists (boards) -// before board_issues. +// convert; the folder's lists (which classify sprints) before tasks; users +// before tasks (issues reference accounts); the folder-board + sprints before +// board_issues/sprint_issues. func (p ClickUp) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ + tasks.CollectListMeta, + tasks.ExtractListMeta, tasks.CollectUserMeta, tasks.ExtractUserMeta, tasks.CollectTaskMeta, tasks.ExtractTaskMeta, - tasks.ConvertListMeta, + tasks.ConvertFolderMeta, + tasks.ConvertSprintMeta, tasks.ConvertUserMeta, tasks.ConvertTaskMeta, } @@ -115,8 +120,8 @@ func (p ClickUp) PrepareTaskData(taskCtx plugin.TaskContext, options map[string] if op.ConnectionId == 0 { return nil, errors.BadInput.New("clickup connectionId is invalid") } - if op.ListId == "" { - return nil, errors.BadInput.New("clickup listId is required") + if op.FolderId == "" { + return nil, errors.BadInput.New("clickup folderId is required") } connection := &models.ClickUpConnection{} diff --git a/backend/plugins/clickup/models/folder.go b/backend/plugins/clickup/models/folder.go new file mode 100644 index 00000000000..347c15e0a21 --- /dev/null +++ b/backend/plugins/clickup/models/folder.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*ClickUpFolder)(nil) + +// ClickUpFolder is the data-source scope for the ClickUp plugin. A ClickUp +// folder (e.g. a team's "Dev Team" / "Sprint Folder") owns the backlog and the +// rolling sprint lists, so it maps cleanly onto a DevLake domain-layer +// ticket.Board — analogous to a Jira board. Selecting the folder (rather than +// an individual list) means new sprints are picked up automatically on each +// sync and ephemeral/archived sprint lists never need re-scoping. +type ClickUpFolder struct { + common.Scope `mapstructure:",squash"` + FolderId string `json:"folderId" mapstructure:"folderId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + SpaceId string `json:"spaceId" mapstructure:"spaceId" gorm:"type:varchar(255)"` + SpaceName string `json:"spaceName" mapstructure:"spaceName" gorm:"type:varchar(255)"` +} + +func (f ClickUpFolder) ScopeId() string { + return f.FolderId +} + +func (f ClickUpFolder) ScopeName() string { + return f.Name +} + +func (f ClickUpFolder) ScopeFullName() string { + if f.SpaceName != "" { + return f.SpaceName + "/" + f.Name + } + return f.Name +} + +func (f ClickUpFolder) ScopeParams() interface{} { + return &ClickUpApiParams{ + ConnectionId: f.ConnectionId, + FolderId: f.FolderId, + } +} + +func (ClickUpFolder) TableName() string { + return "_tool_clickup_folders" +} + +// ClickUpApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_clickup_* table. +type ClickUpApiParams struct { + ConnectionId uint64 + FolderId string +} diff --git a/backend/plugins/clickup/models/list.go b/backend/plugins/clickup/models/list.go index 5b230d91d14..6f60ce710e9 100644 --- a/backend/plugins/clickup/models/list.go +++ b/backend/plugins/clickup/models/list.go @@ -18,52 +18,34 @@ limitations under the License. package models import ( + "time" + "github.com/apache/incubator-devlake/core/models/common" - "github.com/apache/incubator-devlake/core/plugin" ) -var _ plugin.ToolLayerScope = (*ClickUpList)(nil) - -// ClickUpList is the data-source scope for the ClickUp plugin. A ClickUp List -// owns tasks (analogous to a Jira board), mapping cleanly to a DevLake -// domain-layer ticket.Board. +// ClickUpList is a list inside a scoped folder. It is NOT itself a scope (the +// folder is — see ClickUpFolder). A list is one of two things: +// +// - a sprint list (IsSprint) — a rolling sprint, converted to ticket.Sprint; +// tasks in it become sprint_issues. ClickUp teams encode sprints as lists +// named e.g. "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)"; the sprint number and +// start/end dates are parsed from that name (there are no list date fields). +// - a regular list (Backlog / Bug Tracking / DevOps) — its tasks are plain +// board issues, no sprint. type ClickUpList struct { - common.Scope `mapstructure:",squash"` - ListId string `json:"listId" mapstructure:"listId" gorm:"primaryKey;type:varchar(255)"` - Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` - SpaceId string `json:"spaceId" mapstructure:"spaceId" gorm:"type:varchar(255)"` - SpaceName string `json:"spaceName" mapstructure:"spaceName" gorm:"type:varchar(255)"` -} - -func (l ClickUpList) ScopeId() string { - return l.ListId -} - -func (l ClickUpList) ScopeName() string { - return l.Name -} - -func (l ClickUpList) ScopeFullName() string { - if l.SpaceName != "" { - return l.SpaceName + "/" + l.Name - } - return l.Name -} - -func (l ClickUpList) ScopeParams() interface{} { - return &ClickUpApiParams{ - ConnectionId: l.ConnectionId, - ListId: l.ListId, - } + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ListId string `gorm:"primaryKey;type:varchar(255)" json:"listId"` + FolderId string `gorm:"index;type:varchar(255)" json:"folderId"` + SpaceId string `gorm:"type:varchar(255)" json:"spaceId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Archived bool `json:"archived"` + IsSprint bool `json:"isSprint"` + SprintName string `gorm:"type:varchar(255)" json:"sprintName"` + StartDate *time.Time `json:"startDate"` + EndDate *time.Time `json:"endDate"` + common.NoPKModel } func (ClickUpList) TableName() string { return "_tool_clickup_lists" } - -// ClickUpApiParams identifies the scope a raw row belongs to. It is stored in -// the `params` column of every _raw_clickup_* table. -type ClickUpApiParams struct { - ConnectionId uint64 - ListId string -} diff --git a/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go b/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go new file mode 100644 index 00000000000..3f0affbcbd9 --- /dev/null +++ b/backend/plugins/clickup/models/migrationscripts/20260722_add_folder_scope.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addFolderScope moves the plugin's data-source scope from List to Folder +// (Jira-parity: folder = board). It creates _tool_clickup_folders and adds the +// sprint/story-point/folder columns to the existing tool tables. AutoMigrate is +// additive: pre-existing columns from the list-scope schema are left in place. + +// frozen snapshots for this migration (suffixed to avoid clashing with the +// 20260720 archived snapshots that share the same table names). + +type clickUpFolder20260722 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeConfigId uint64 + FolderId string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + SpaceName string `gorm:"type:varchar(255)"` +} + +func (clickUpFolder20260722) TableName() string { return "_tool_clickup_folders" } + +type clickUpList20260722 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ListId string `gorm:"primaryKey;type:varchar(255)"` + FolderId string `gorm:"index;type:varchar(255)"` + SpaceId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Archived bool + IsSprint bool + SprintName string `gorm:"type:varchar(255)"` + StartDate *time.Time + EndDate *time.Time +} + +func (clickUpList20260722) TableName() string { return "_tool_clickup_lists" } + +type clickUpTask20260722 struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + FolderId string `gorm:"index;type:varchar(255)"` + StoryPoint *float64 `gorm:"column:story_point"` + archived.NoPKModel +} + +func (clickUpTask20260722) TableName() string { return "_tool_clickup_tasks" } + +type clickUpScopeConfig20260722 struct { + archived.ScopeConfig + SprintNamePattern string `gorm:"type:varchar(255)"` + StoryPointField string `gorm:"type:varchar(255)"` + DefaultIssueType string `gorm:"type:varchar(100)"` +} + +func (clickUpScopeConfig20260722) TableName() string { return "_tool_clickup_scope_configs" } + +type addFolderScope struct{} + +func (*addFolderScope) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &clickUpFolder20260722{}, + &clickUpList20260722{}, + &clickUpTask20260722{}, + &clickUpScopeConfig20260722{}, + ) +} + +func (*addFolderScope) Version() uint64 { + return 20260722000001 +} + +func (*addFolderScope) Name() string { + return "clickup: add folder scope + sprint/story-point columns" +} diff --git a/backend/plugins/clickup/models/migrationscripts/register.go b/backend/plugins/clickup/models/migrationscripts/register.go index ec054748c27..05fed213a2f 100644 --- a/backend/plugins/clickup/models/migrationscripts/register.go +++ b/backend/plugins/clickup/models/migrationscripts/register.go @@ -25,5 +25,6 @@ import ( func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), + new(addFolderScope), } } diff --git a/backend/plugins/clickup/models/scope_config.go b/backend/plugins/clickup/models/scope_config.go index 34988e26e0c..ab30136462f 100644 --- a/backend/plugins/clickup/models/scope_config.go +++ b/backend/plugins/clickup/models/scope_config.go @@ -44,6 +44,20 @@ type ClickUpScopeConfig struct { IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` + // SprintNamePattern is a regex identifying which lists in the folder are + // sprint lists (converted to ticket.Sprint; their tasks become + // sprint_issues). Lists not matching are plain board issues. Empty -> + // defaultSprintNamePattern. + SprintNamePattern string `mapstructure:"sprintNamePattern,omitempty" json:"sprintNamePattern" gorm:"type:varchar(255)"` + // StoryPointField selects where a task's story points come from. Empty + // defaults to ClickUp's native sprint points field ("points"). Set to a + // custom-field name to read Fibonacci/LOE from a custom field instead. + StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"` + // DefaultIssueType, when set (REQUIREMENT/BUG/INCIDENT), forces every issue + // on this board to that type — used to flag a whole folder as incidents + // (e.g. the "Security Incidents & Response" folder feeding DORA CFR/MTTR). + // Empty -> per-task type detection (IssueType* patterns) applies. + DefaultIssueType string `mapstructure:"defaultIssueType,omitempty" json:"defaultIssueType" gorm:"type:varchar(100)"` } func (ClickUpScopeConfig) TableName() string { diff --git a/backend/plugins/clickup/models/task.go b/backend/plugins/clickup/models/task.go index 60588e9ae99..957889edffb 100644 --- a/backend/plugins/clickup/models/task.go +++ b/backend/plugins/clickup/models/task.go @@ -28,6 +28,7 @@ type ClickUpTask struct { ConnectionId uint64 `gorm:"primaryKey"` Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` ListId string `gorm:"index;type:varchar(255)" json:"listId"` + FolderId string `gorm:"index;type:varchar(255)" json:"folderId"` SpaceId string `gorm:"type:varchar(255)" json:"spaceId"` CustomId string `gorm:"type:varchar(255)" json:"customId"` Name string `json:"name"` @@ -41,6 +42,7 @@ type ClickUpTask struct { AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` AssigneeName string `gorm:"type:varchar(255)" json:"assigneeName"` ParentId string `gorm:"type:varchar(255)" json:"parentId"` + StoryPoint *float64 `json:"storyPoint"` CreatedDate *time.Time `json:"createdDate"` UpdatedDate *time.Time `gorm:"index" json:"updatedDate"` ClosedDate *time.Time `json:"closedDate"` diff --git a/backend/plugins/clickup/tasks/board_convertor.go b/backend/plugins/clickup/tasks/folder_convertor.go similarity index 64% rename from backend/plugins/clickup/tasks/board_convertor.go rename to backend/plugins/clickup/tasks/folder_convertor.go index bb82a4e1b84..e5306ef9ca4 100644 --- a/backend/plugins/clickup/tasks/board_convertor.go +++ b/backend/plugins/clickup/tasks/folder_convertor.go @@ -30,34 +30,35 @@ import ( "github.com/apache/incubator-devlake/plugins/clickup/models" ) -// RAW_LIST_TABLE labels the raw-data lineage for the list-scope-derived board. -// Lists are added as scopes (no collector), so this is a logical tag only. -const RAW_LIST_TABLE = "clickup_lists" +// RAW_FOLDER_TABLE labels the raw-data lineage for the folder-scope-derived +// board. The folder is added as a scope (no collector), so this is a logical +// tag only. +const RAW_FOLDER_TABLE = "clickup_folders" -var ConvertListMeta = plugin.SubTaskMeta{ - Name: "Convert Lists", - EntryPoint: ConvertLists, +var ConvertFolderMeta = plugin.SubTaskMeta{ + Name: "Convert Folders", + EntryPoint: ConvertFolders, EnabledByDefault: true, - Description: "Convert the ClickUp list scope (_tool_clickup_lists) into the domain layer table boards", + Description: "Convert the ClickUp folder scope (_tool_clickup_folders) into the domain layer table boards", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, - DependencyTables: []string{models.ClickUpList{}.TableName()}, + DependencyTables: []string{models.ClickUpFolder{}.TableName()}, ProductTables: []string{ticket.Board{}.TableName()}, } -var _ plugin.SubTaskEntryPoint = ConvertLists +var _ plugin.SubTaskEntryPoint = ConvertFolders -func ConvertLists(taskCtx plugin.SubTaskContext) errors.Error { +func ConvertFolders(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*ClickUpTaskData) connectionId := data.Options.ConnectionId - // boardId must be generated identically to the task convertor so the board - // joins to the board_issues that reference it. - boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + // boardId must be generated identically to the task/sprint convertors so the + // board joins to the board_issues/board_sprints that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) cursor, err := db.Cursor( - dal.From(&models.ClickUpList{}), - dal.Where("connection_id = ? AND list_id = ?", connectionId, data.Options.ListId), + dal.From(&models.ClickUpFolder{}), + dal.Where("connection_id = ? AND folder_id = ?", connectionId, data.Options.FolderId), ) if err != nil { return err @@ -69,17 +70,17 @@ func ConvertLists(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: connectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, - Table: RAW_LIST_TABLE, + Table: RAW_FOLDER_TABLE, }, - InputRowType: reflect.TypeOf(models.ClickUpList{}), + InputRowType: reflect.TypeOf(models.ClickUpFolder{}), Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - list := inputRow.(*models.ClickUpList) + folder := inputRow.(*models.ClickUpFolder) board := &ticket.Board{ - DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, list.ListId)}, - Name: list.ScopeFullName(), + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, folder.FolderId)}, + Name: folder.ScopeFullName(), Type: "clickup", } return []interface{}{board}, nil diff --git a/backend/plugins/clickup/tasks/list_collector.go b/backend/plugins/clickup/tasks/list_collector.go new file mode 100644 index 00000000000..bc601f68218 --- /dev/null +++ b/backend/plugins/clickup/tasks/list_collector.go @@ -0,0 +1,111 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// RAW_LIST_TABLE holds the raw lists of a scoped folder (GET /folder/{id}/list). +const RAW_LIST_TABLE = "clickup_lists" + +// clickUpListResponse mirrors GET /folder/{id}/list. The endpoint is not +// paginated and returns the folder's lists under `lists`. +type clickUpListResponse struct { + Lists []json.RawMessage `json:"lists"` +} + +// archivedInput drives the collector to request the folder's lists twice: once +// for active lists and once for archived lists (ClickUp's /folder/{id}/list +// returns active by default and archived only when archived=true). +type archivedInput struct { + Archived bool +} + +// archivedIterator yields {false, true} so a single Execute collects both the +// active and the archived lists into the same raw table. +type archivedIterator struct { + vals []bool + i int +} + +func newArchivedIterator() *archivedIterator { return &archivedIterator{vals: []bool{false, true}} } + +func (it *archivedIterator) HasNext() bool { return it.i < len(it.vals) } + +func (it *archivedIterator) Fetch() (interface{}, errors.Error) { + v := it.vals[it.i] + it.i++ + return &archivedInput{Archived: v}, nil +} + +func (it *archivedIterator) Close() errors.Error { return nil } + +var CollectListMeta = plugin.SubTaskMeta{ + Name: "Collect Lists", + EntryPoint: CollectLists, + EnabledByDefault: true, + Description: "Collect the lists (backlog + sprint lists, active and archived) of a scoped ClickUp folder", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectLists + +func CollectLists(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + ApiClient: data.ApiClient, + Input: newArchivedIterator(), + UrlTemplate: "folder/{{ .Params.FolderId }}/list", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + archived := false + if in, ok := reqData.Input.(*archivedInput); ok { + archived = in.Archived + } + query.Set("archived", strconv.FormatBool(archived)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp clickUpListResponse + if err := api.UnmarshalResponse(res, &resp); err != nil { + return nil, err + } + return resp.Lists, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/clickup/tasks/list_extractor.go b/backend/plugins/clickup/tasks/list_extractor.go new file mode 100644 index 00000000000..01d18563e45 --- /dev/null +++ b/backend/plugins/clickup/tasks/list_extractor.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +var ExtractListMeta = plugin.SubTaskMeta{ + Name: "Extract Lists", + EntryPoint: ExtractLists, + EnabledByDefault: true, + Description: "Extract raw folder lists into _tool_clickup_lists, classifying sprint lists", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractLists + +// ClickUpApiList is the subset of a ClickUp list JSON the extractor reads. +type ClickUpApiList struct { + Id string `json:"id"` + Name string `json:"name"` + Archived bool `json:"archived"` + Folder *struct { + Id string `json:"id"` + } `json:"folder"` + Space *struct { + Id string `json:"id"` + } `json:"space"` +} + +func ExtractLists(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ClickUpTaskData) + detector, err := newSprintDetector(data.ScopeConfig) + if err != nil { + return err + } + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: data.Options.ConnectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiList := &ClickUpApiList{} + if err := errors.Convert(json.Unmarshal(row.Data, apiList)); err != nil { + return nil, err + } + if apiList.Id == "" { + return nil, nil + } + folderId := data.Options.FolderId + if apiList.Folder != nil && apiList.Folder.Id != "" { + folderId = apiList.Folder.Id + } + list := &models.ClickUpList{ + ConnectionId: data.Options.ConnectionId, + ListId: apiList.Id, + FolderId: folderId, + Name: apiList.Name, + Archived: apiList.Archived, + } + if apiList.Space != nil { + list.SpaceId = apiList.Space.Id + } + if sprint := detector.detect(apiList.Name); sprint != nil { + list.IsSprint = true + list.SprintName = sprint.name + list.StartDate = sprint.start + list.EndDate = sprint.end + } + return []interface{}{list}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/clickup/tasks/shared.go b/backend/plugins/clickup/tasks/shared.go index 64d41df5c67..783c3c6e79f 100644 --- a/backend/plugins/clickup/tasks/shared.go +++ b/backend/plugins/clickup/tasks/shared.go @@ -45,6 +45,88 @@ func parseClickUpTime(ms string) *time.Time { return &t } +// defaultSprintNamePattern identifies sprint lists by name. ClickUp teams name +// sprint lists like "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)" or "Sprint 31 (7/6 - +// 7/19)", so any list whose name contains "Sprint " is treated as a sprint. +const defaultSprintNamePattern = `(?i)sprint\s*\d+` + +// sprintDateRange captures the "(start - end)" span embedded in a sprint list +// name. Group 1 = start token, group 2 = end token (each m/d[/yy] or d/m[/yy]). +var sprintDateRange = regexp.MustCompile(`\(\s*([\d/]+)\s*-\s*([\d/]+)\s*\)`) + +type sprintInfo struct { + name string + start *time.Time + end *time.Time +} + +// sprintDetector classifies a list name as a sprint (or not) using the scope +// config's SprintNamePattern (default defaultSprintNamePattern). +type sprintDetector struct { + re *regexp.Regexp +} + +func newSprintDetector(sc *models.ClickUpScopeConfig) (*sprintDetector, errors.Error) { + pattern := defaultSprintNamePattern + if sc != nil && sc.SprintNamePattern != "" { + pattern = sc.SprintNamePattern + } + re, err := errors.Convert01(regexp.Compile(pattern)) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid sprintNamePattern") + } + return &sprintDetector{re: re}, nil +} + +// detect returns sprint info for a matching list name, or nil for a non-sprint +// (backlog / bug / other) list. +func (d *sprintDetector) detect(name string) *sprintInfo { + if !d.re.MatchString(name) { + return nil + } + start, end := parseSprintDates(name) + return &sprintInfo{name: name, start: start, end: end} +} + +// parseSprintDates extracts the start/end dates from a sprint list name's +// "(m/d/yy - m/d/yy)" span. Teams differ on ordering (Toto uses M/D/YY, +// Blockchain D/M/YY), so the order is disambiguated per token: a component > 12 +// must be the day. Genuinely ambiguous tokens (both <= 12) default to M/D. +// Tokens without a year are left unset (nil) rather than guessed. +func parseSprintDates(name string) (*time.Time, *time.Time) { + m := sprintDateRange.FindStringSubmatch(name) + if m == nil { + return nil, nil + } + return parseSprintDate(m[1]), parseSprintDate(m[2]) +} + +func parseSprintDate(token string) *time.Time { + parts := strings.Split(strings.TrimSpace(token), "/") + if len(parts) != 3 { + // No year component -> cannot form a reliable date. + return nil + } + a, errA := strconv.Atoi(parts[0]) + b, errB := strconv.Atoi(parts[1]) + y, errY := strconv.Atoi(parts[2]) + if errA != nil || errB != nil || errY != nil { + return nil + } + month, day := a, b + if a > 12 { // first component can't be a month -> D/M ordering + month, day = b, a + } + if month < 1 || month > 12 || day < 1 || day > 31 { + return nil + } + if y < 100 { + y += 2000 + } + t := time.Date(y, time.Month(month), day, 0, 0, 0, 0, time.UTC) + return &t +} + // statusFromType maps a ClickUp status.type onto a DevLake standard issue // status. ClickUp's status types are standardized: // diff --git a/backend/plugins/clickup/tasks/shared_test.go b/backend/plugins/clickup/tasks/shared_test.go new file mode 100644 index 00000000000..a84fb1a693b --- /dev/null +++ b/backend/plugins/clickup/tasks/shared_test.go @@ -0,0 +1,254 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +func TestParseSprintDate(t *testing.T) { + tests := []struct { + name string + token string + want string // "" means nil (no reliable date) + }{ + {"Toto M/D/YY", "7/6/26", "2026-07-06"}, + {"Toto M/D/YY day>12", "7/19/26", "2026-07-19"}, + {"Blockchain D/M/YY day>12", "29/6/26", "2026-06-29"}, + {"Blockchain D/M/YY", "26/7/26", "2026-07-26"}, + {"ambiguous both<=12 defaults M/D", "3/4/26", "2026-03-04"}, + {"no year -> nil", "7/6", ""}, + {"empty -> nil", "", ""}, + {"invalid month both>12 -> nil", "13/13/26", ""}, + {"non-numeric -> nil", "a/b/c", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseSprintDate(tt.token) + if tt.want == "" { + if got != nil { + t.Fatalf("expected nil, got %v", got) + } + return + } + if got == nil { + t.Fatalf("expected %s, got nil", tt.want) + } + if g := got.Format("2006-01-02"); g != tt.want { + t.Fatalf("expected %s, got %s", tt.want, g) + } + }) + } +} + +func TestSprintDetector(t *testing.T) { + det, err := newSprintDetector(nil) // nil -> default pattern + if err != nil { + t.Fatalf("newSprintDetector: %v", err) + } + tests := []struct { + name string + listName string + wantSprint bool + wantStart string + }{ + {"toto sprint with dates", "v4.3.0 Sprint 40 (7/6/26 - 7/19/26)", true, "2026-07-06"}, + {"blockchain sprint D/M", "Sprint 25 (29/6/26 - 26/7/26)", true, "2026-06-29"}, + {"sprint no year -> no dates", "Sprint 31 (7/6 - 7/19)", true, ""}, + {"backlog is not a sprint", "Backlog", false, ""}, + {"qa bugs is not a sprint", "QA Bugs", false, ""}, + {"devops is not a sprint", "DevOps", false, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := det.detect(tt.listName) + if tt.wantSprint != (got != nil) { + t.Fatalf("wantSprint=%v, got=%v", tt.wantSprint, got != nil) + } + if got == nil { + return + } + if got.name != tt.listName { + t.Fatalf("sprint name = %q, want %q", got.name, tt.listName) + } + if tt.wantStart == "" { + if got.start != nil { + t.Fatalf("expected nil start, got %v", got.start) + } + } else if got.start == nil || got.start.Format("2006-01-02") != tt.wantStart { + t.Fatalf("start = %v, want %s", got.start, tt.wantStart) + } + }) + } +} + +func TestStatusFromType(t *testing.T) { + tests := []struct { + statusType string + want string + }{ + {"open", ticket.TODO}, + {"unstarted", ticket.TODO}, + {"custom", ticket.IN_PROGRESS}, + {"done", ticket.DONE}, + {"closed", ticket.DONE}, + {"CLOSED", ticket.DONE}, // case-insensitive + {"mystery", ticket.OTHER}, + } + for _, tt := range tests { + t.Run(tt.statusType, func(t *testing.T) { + if got := statusFromType(tt.statusType); got != tt.want { + t.Fatalf("statusFromType(%q) = %q, want %q", tt.statusType, got, tt.want) + } + }) + } +} + +func TestStatusMapperOverride(t *testing.T) { + // Toto tags "to do" and "re-open" as ClickUp type "custom", which the + // type-default would file as IN_PROGRESS. The scope-config override must win. + sc := &models.ClickUpScopeConfig{ + IssueStatusTodo: []string{"to do", "re-open"}, + IssueStatusInProgress: []string{"in development"}, + IssueStatusDone: []string{"deployed", "Closed"}, + } + m := newStatusMapper(sc) + tests := []struct { + rawStatus string + statusType string + want string + }{ + {"to do", "custom", ticket.TODO}, // override beats type default + {"re-open", "custom", ticket.TODO}, // override beats type default + {"in development", "custom", ticket.IN_PROGRESS}, + {"deployed", "done", ticket.DONE}, + {"CLOSED", "closed", ticket.DONE}, // override match is case-insensitive + {"in code review", "custom", ticket.IN_PROGRESS}, // not listed -> type default + } + for _, tt := range tests { + t.Run(tt.rawStatus, func(t *testing.T) { + if got := m.statusOf(tt.rawStatus, tt.statusType); got != tt.want { + t.Fatalf("statusOf(%q,%q) = %q, want %q", tt.rawStatus, tt.statusType, got, tt.want) + } + }) + } +} + +func TestStatusMapperNoConfigFallsBackToType(t *testing.T) { + m := newStatusMapper(nil) + if got := m.statusOf("anything", "open"); got != ticket.TODO { + t.Fatalf("expected TODO from type, got %q", got) + } +} + +func TestIssueTypeMatcher(t *testing.T) { + sc := &models.ClickUpScopeConfig{ + IssueTypeIncident: "(?i)incident", + IssueTypeBug: "(?i)^bug$", + IssueTypeRequirement: "(?i)(feature|story)", + } + m, err := newIssueTypeMatcher(sc) + if err != nil { + t.Fatalf("newIssueTypeMatcher: %v", err) + } + tests := []struct { + taskType string + want string + }{ + {"incident", ticket.INCIDENT}, + {"Bug", ticket.BUG}, + {"feature", ticket.REQUIREMENT}, + {"", ticket.REQUIREMENT}, // no match -> default REQUIREMENT + } + for _, tt := range tests { + t.Run(tt.taskType, func(t *testing.T) { + if got := m.typeOf(tt.taskType); got != tt.want { + t.Fatalf("typeOf(%q) = %q, want %q", tt.taskType, got, tt.want) + } + }) + } +} + +func TestIssueTypeMatcherDefaultBug(t *testing.T) { + // nil config still classifies ClickUp's built-in "Bug" via defaultBugPattern. + m, err := newIssueTypeMatcher(nil) + if err != nil { + t.Fatalf("newIssueTypeMatcher: %v", err) + } + if got := m.typeOf("bug"); got != ticket.BUG { + t.Fatalf("typeOf(bug) = %q, want BUG", got) + } + if got := m.typeOf("anything-else"); got != ticket.REQUIREMENT { + t.Fatalf("typeOf(anything-else) = %q, want REQUIREMENT", got) + } +} + +func TestStoryPointOf(t *testing.T) { + fp := func(f float64) *float64 { return &f } + native := &ClickUpApiTask{Points: fp(8)} + // default (no field configured) uses native points + if got := storyPointOf(native, nil); got == nil || *got != 8 { + t.Fatalf("native points: got %v, want 8", got) + } + // configured custom field wins + custom := &ClickUpApiTask{ + Points: fp(8), + CustomFields: []clickUpCustomField{ + {Name: "LOE", Value: json.RawMessage(`"13"`)}, + }, + } + sc := &models.ClickUpScopeConfig{StoryPointField: "loe"} // case-insensitive + if got := storyPointOf(custom, sc); got == nil || *got != 13 { + t.Fatalf("custom field: got %v, want 13", got) + } + // configured field absent on task -> nil (does not fall back to native) + if got := storyPointOf(native, sc); got != nil { + t.Fatalf("missing custom field: got %v, want nil", got) + } +} + +func TestNumericValue(t *testing.T) { + tests := []struct { + raw string + want *float64 + }{ + {`5`, ptr(5)}, + {`"8"`, ptr(8)}, + {`3.5`, ptr(3.5)}, + {``, nil}, + {`null`, nil}, + {`"abc"`, nil}, + } + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got := numericValue(json.RawMessage(tt.raw)) + if (tt.want == nil) != (got == nil) { + t.Fatalf("raw %q: got %v, want %v", tt.raw, got, tt.want) + } + if tt.want != nil && *got != *tt.want { + t.Fatalf("raw %q: got %v, want %v", tt.raw, *got, *tt.want) + } + }) + } +} + +func ptr(f float64) *float64 { return &f } diff --git a/backend/plugins/clickup/tasks/sprint_convertor.go b/backend/plugins/clickup/tasks/sprint_convertor.go new file mode 100644 index 00000000000..b5849882e3c --- /dev/null +++ b/backend/plugins/clickup/tasks/sprint_convertor.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" +) + +// Sprint status values follow the convention used by the Jira plugin. +const ( + sprintStatusFuture = "FUTURE" + sprintStatusActive = "ACTIVE" + sprintStatusClosed = "CLOSED" +) + +var ConvertSprintMeta = plugin.SubTaskMeta{ + Name: "Convert Sprints", + EntryPoint: ConvertSprints, + EnabledByDefault: true, + Description: "Convert sprint lists (_tool_clickup_lists where is_sprint) into domain tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.ClickUpList{}.TableName()}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprints + +func ConvertSprints(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ClickUpTaskData) + connectionId := data.Options.ConnectionId + + sprintIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + boardId := boardIdGen.Generate(connectionId, data.Options.FolderId) + now := time.Now() + + cursor, err := db.Cursor( + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ? AND is_sprint = ?", connectionId, data.Options.FolderId, true), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ClickUpApiParams{ + ConnectionId: connectionId, + FolderId: data.Options.FolderId, + }, + Table: RAW_LIST_TABLE, + }, + InputRowType: reflect.TypeOf(models.ClickUpList{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + list := inputRow.(*models.ClickUpList) + sprintId := sprintIdGen.Generate(connectionId, list.ListId) + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: list.SprintName, + Status: sprintStatus(list, now), + StartedDate: list.StartDate, + EndedDate: list.EndDate, + OriginalBoardID: boardId, + } + if sprint.Status == sprintStatusClosed { + sprint.CompletedDate = list.EndDate + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// sprintStatus derives a sprint's lifecycle from its dates and archived flag. +// Archived sprint lists are always closed; otherwise the current time relative +// to the parsed start/end window decides. Missing dates default to ACTIVE. +func sprintStatus(list *models.ClickUpList, now time.Time) string { + if list.Archived { + return sprintStatusClosed + } + if list.EndDate != nil && now.After(*list.EndDate) { + return sprintStatusClosed + } + if list.StartDate != nil && now.Before(*list.StartDate) { + return sprintStatusFuture + } + return sprintStatusActive +} diff --git a/backend/plugins/clickup/tasks/task_collector.go b/backend/plugins/clickup/tasks/task_collector.go index afc92d99e76..8d2572e5f73 100644 --- a/backend/plugins/clickup/tasks/task_collector.go +++ b/backend/plugins/clickup/tasks/task_collector.go @@ -21,28 +21,37 @@ import ( "encoding/json" "net/http" "net/url" + "reflect" "strconv" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/clickup/models" ) const RAW_TASK_TABLE = "clickup_tasks" -// clickUpTaskListResponse mirrors the envelope returned by -// GET /list/{id}/task. ClickUp uses zero-based page numbers and signals the end -// of the collection with `last_page: true` (and/or an empty `tasks` array). +// clickUpTaskListResponse mirrors the envelope returned by GET /list/{id}/task. +// ClickUp uses zero-based page numbers and signals the end of the collection +// with `last_page: true` (and/or an empty `tasks` array). type clickUpTaskListResponse struct { Tasks []json.RawMessage `json:"tasks"` LastPage bool `json:"last_page"` } +// listInput is the per-list iterator element driving task collection: tasks are +// collected list-by-list across every list in the scoped folder. +type listInput struct { + ListId string `gorm:"column:list_id"` +} + var CollectTaskMeta = plugin.SubTaskMeta{ Name: "Collect Tasks", EntryPoint: CollectTasks, EnabledByDefault: true, - Description: "Collect tasks for a ClickUp list (page-based pagination)", + Description: "Collect tasks for every list in the scoped ClickUp folder (page-based pagination)", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } @@ -50,18 +59,35 @@ var _ plugin.SubTaskEntryPoint = CollectTasks func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*ClickUpTaskData) + db := taskCtx.GetDal() + + // Iterate every list of this folder collected by CollectLists. + cursor, err := db.Cursor( + dal.Select("list_id"), + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ?", data.Options.ConnectionId, data.Options.FolderId), + ) + if err != nil { + return err + } + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(listInput{})) + if err != nil { + return err + } + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: data.Options.ConnectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_TASK_TABLE, }, ApiClient: data.ApiClient, + Input: iterator, PageSize: 100, - UrlTemplate: "list/{{ .Params.ListId }}/task", + UrlTemplate: "list/{{ .Input.ListId }}/task", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("subtasks", "true") diff --git a/backend/plugins/clickup/tasks/task_convertor.go b/backend/plugins/clickup/tasks/task_convertor.go index b779019144c..24fe007b73f 100644 --- a/backend/plugins/clickup/tasks/task_convertor.go +++ b/backend/plugins/clickup/tasks/task_convertor.go @@ -19,6 +19,7 @@ package tasks import ( "reflect" + "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -34,10 +35,10 @@ var ConvertTaskMeta = plugin.SubTaskMeta{ Name: "Convert Tasks", EntryPoint: ConvertTasks, EnabledByDefault: true, - Description: "Convert tool layer table _tool_clickup_tasks into domain layer tables issues and board_issues", + Description: "Convert tool layer table _tool_clickup_tasks into domain layer tables issues, board_issues and sprint_issues", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, - DependencyTables: []string{models.ClickUpTask{}.TableName(), RAW_TASK_TABLE}, - ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, + DependencyTables: []string{models.ClickUpTask{}.TableName(), models.ClickUpList{}.TableName(), RAW_TASK_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.SprintIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, } var _ plugin.SubTaskEntryPoint = ConvertTasks @@ -49,18 +50,31 @@ func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { issueIdGen := didgen.NewDomainIdGenerator(&models.ClickUpTask{}) accountIdGen := didgen.NewDomainIdGenerator(&models.ClickUpUser{}) - boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) - boardId := boardIdGen.Generate(connectionId, data.Options.ListId) + boardIdGen := didgen.NewDomainIdGenerator(&models.ClickUpFolder{}) + sprintIdGen := didgen.NewDomainIdGenerator(&models.ClickUpList{}) + boardId := boardIdGen.Generate(connectionId, data.Options.FolderId) + + // Set of sprint list ids so a task in a sprint list also produces a + // sprint_issue (velocity/throughput). Non-sprint lists (Backlog / Bug + // Tracking) contribute only board_issues. + sprintListIds, err := loadSprintListIds(db, connectionId, data.Options.FolderId) + if err != nil { + return err + } statusMapper := newStatusMapper(data.ScopeConfig) typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) if err != nil { return err } + defaultType := "" + if data.ScopeConfig != nil && data.ScopeConfig.DefaultIssueType != "" { + defaultType = strings.ToUpper(strings.TrimSpace(data.ScopeConfig.DefaultIssueType)) + } cursor, err := db.Cursor( dal.From(&models.ClickUpTask{}), - dal.Where("connection_id = ? AND list_id = ?", connectionId, data.Options.ListId), + dal.Where("connection_id = ? AND folder_id = ?", connectionId, data.Options.FolderId), ) if err != nil { return err @@ -72,7 +86,7 @@ func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: connectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_TASK_TABLE, }, @@ -86,17 +100,23 @@ func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { issueKey = task.Id } + issueType := typeMatcher.typeOf(task.Type) + if defaultType != "" { + issueType = defaultType + } + domainIssue := &ticket.Issue{ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, task.Id)}, IssueKey: issueKey, Title: task.Name, Description: task.Description, Url: task.Url, - Type: typeMatcher.typeOf(task.Type), + Type: issueType, OriginalType: task.Type, Status: statusMapper.statusOf(task.Status, task.StatusType), OriginalStatus: task.Status, Priority: task.Priority, + StoryPoint: task.StoryPoint, CreatedDate: task.CreatedDate, UpdatedDate: task.UpdatedDate, ResolutionDate: task.ClosedDate, @@ -121,11 +141,16 @@ func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { domainIssue.LeadTimeMinutes = &minutes } - boardIssue := &ticket.BoardIssue{ - BoardId: boardId, - IssueId: domainIssue.Id, + results := []interface{}{ + domainIssue, + &ticket.BoardIssue{BoardId: boardId, IssueId: domainIssue.Id}, + } + if task.ListId != "" && sprintListIds[task.ListId] { + results = append(results, &ticket.SprintIssue{ + SprintId: sprintIdGen.Generate(connectionId, task.ListId), + IssueId: domainIssue.Id, + }) } - results := []interface{}{domainIssue, boardIssue} if domainIssue.AssigneeId != "" { results = append(results, &ticket.IssueAssignee{ IssueId: domainIssue.Id, @@ -141,3 +166,21 @@ func ConvertTasks(taskCtx plugin.SubTaskContext) errors.Error { } return converter.Execute() } + +// loadSprintListIds returns the set of list ids in the folder that are sprint +// lists, so the task convertor can emit sprint_issues for their tasks. +func loadSprintListIds(db dal.Dal, connectionId uint64, folderId string) (map[string]bool, errors.Error) { + var lists []models.ClickUpList + if err := db.All(&lists, + dal.Select("list_id"), + dal.From(&models.ClickUpList{}), + dal.Where("connection_id = ? AND folder_id = ? AND is_sprint = ?", connectionId, folderId, true), + ); err != nil { + return nil, err + } + ids := make(map[string]bool, len(lists)) + for _, l := range lists { + ids[l.ListId] = true + } + return ids, nil +} diff --git a/backend/plugins/clickup/tasks/task_data.go b/backend/plugins/clickup/tasks/task_data.go index 8b6bd0f9719..698e42b7220 100644 --- a/backend/plugins/clickup/tasks/task_data.go +++ b/backend/plugins/clickup/tasks/task_data.go @@ -27,7 +27,7 @@ import ( // ClickUpOptions are the per-scope options passed to a pipeline task. type ClickUpOptions struct { ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` - ListId string `json:"listId" mapstructure:"listId,omitempty"` + FolderId string `json:"folderId" mapstructure:"folderId,omitempty"` ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` // TimeAfter limits collection to data created/updated after this time. TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` diff --git a/backend/plugins/clickup/tasks/task_extractor.go b/backend/plugins/clickup/tasks/task_extractor.go index 5bee7ecff28..f3d1209c702 100644 --- a/backend/plugins/clickup/tasks/task_extractor.go +++ b/backend/plugins/clickup/tasks/task_extractor.go @@ -19,6 +19,8 @@ package tasks import ( "encoding/json" + "strconv" + "strings" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -61,10 +63,14 @@ type ClickUpApiTask struct { Priority *struct { Priority string `json:"priority"` } `json:"priority"` - Parent string `json:"parent"` - Url string `json:"url"` - TaskType string `json:"task_type"` - List *struct { + Parent string `json:"parent"` + Url string `json:"url"` + // Points is ClickUp's native sprint points field (Fibonacci LOE for these + // teams). It is the default story-point source. + Points *float64 `json:"points"` + CustomFields []clickUpCustomField `json:"custom_fields"` + TaskType string `json:"task_type"` + List *struct { Id string `json:"id"` } `json:"list"` Space *struct { @@ -72,6 +78,13 @@ type ClickUpApiTask struct { } `json:"space"` } +// clickUpCustomField is one entry of a task's custom_fields array. Value is left +// raw because ClickUp encodes it as a number or a string depending on the field. +type clickUpCustomField struct { + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*ClickUpTaskData) extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ @@ -79,7 +92,7 @@ func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: data.Options.ConnectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_TASK_TABLE, }, @@ -95,19 +108,21 @@ func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { if description == "" { description = apiTask.TextContent } - listId := data.Options.ListId - if apiTask.List != nil && apiTask.List.Id != "" { + listId := "" + if apiTask.List != nil { listId = apiTask.List.Id } task := &models.ClickUpTask{ ConnectionId: data.Options.ConnectionId, Id: apiTask.Id, ListId: listId, + FolderId: data.Options.FolderId, CustomId: apiTask.CustomId, Name: apiTask.Name, Description: description, Type: apiTask.TaskType, ParentId: parentOf(apiTask.Parent), + StoryPoint: storyPointOf(apiTask, data.ScopeConfig), Url: apiTask.Url, CreatedDate: parseClickUpTime(apiTask.DateCreated), UpdatedDate: parseClickUpTime(apiTask.DateUpdated), @@ -150,3 +165,32 @@ func parentOf(parent string) string { } return parent } + +// storyPointOf resolves a task's story points. Default source is ClickUp's +// native sprint points field. When the scope config names a custom field, that +// field's numeric value wins (teams that track Fibonacci LOE in a custom field). +func storyPointOf(apiTask *ClickUpApiTask, sc *models.ClickUpScopeConfig) *float64 { + if sc != nil && sc.StoryPointField != "" { + for _, cf := range apiTask.CustomFields { + if strings.EqualFold(cf.Name, sc.StoryPointField) { + return numericValue(cf.Value) + } + } + return nil + } + return apiTask.Points +} + +// numericValue coerces a raw custom-field value (JSON number or quoted string) +// into a float pointer; returns nil for empty / non-numeric values. +func numericValue(raw json.RawMessage) *float64 { + s := strings.TrimSpace(strings.Trim(string(raw), `"`)) + if s == "" || s == "null" { + return nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil + } + return &f +} diff --git a/backend/plugins/clickup/tasks/user_collector.go b/backend/plugins/clickup/tasks/user_collector.go index 421b598f95c..47933cdc39d 100644 --- a/backend/plugins/clickup/tasks/user_collector.go +++ b/backend/plugins/clickup/tasks/user_collector.go @@ -51,12 +51,12 @@ func CollectUsers(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: data.Options.ConnectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_USER_TABLE, }, ApiClient: data.ApiClient, - UrlTemplate: "list/{{ .Params.ListId }}/member", + UrlTemplate: "folder/{{ .Params.FolderId }}/member", ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var resp clickUpMemberListResponse if err := api.UnmarshalResponse(res, &resp); err != nil { diff --git a/backend/plugins/clickup/tasks/user_convertor.go b/backend/plugins/clickup/tasks/user_convertor.go index 1ff4f0c3d6d..f8c91582b58 100644 --- a/backend/plugins/clickup/tasks/user_convertor.go +++ b/backend/plugins/clickup/tasks/user_convertor.go @@ -61,7 +61,7 @@ func ConvertUsers(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: data.Options.ConnectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_USER_TABLE, }, diff --git a/backend/plugins/clickup/tasks/user_extractor.go b/backend/plugins/clickup/tasks/user_extractor.go index d35aa811d66..12f66c5639d 100644 --- a/backend/plugins/clickup/tasks/user_extractor.go +++ b/backend/plugins/clickup/tasks/user_extractor.go @@ -52,7 +52,7 @@ func ExtractUsers(taskCtx plugin.SubTaskContext) errors.Error { Ctx: taskCtx, Params: ClickUpApiParams{ ConnectionId: data.Options.ConnectionId, - ListId: data.Options.ListId, + FolderId: data.Options.FolderId, }, Table: RAW_USER_TABLE, }, diff --git a/config-ui/src/plugins/components/scope-config-form/index.tsx b/config-ui/src/plugins/components/scope-config-form/index.tsx index 46fbf129a59..8b88385f432 100644 --- a/config-ui/src/plugins/components/scope-config-form/index.tsx +++ b/config-ui/src/plugins/components/scope-config-form/index.tsx @@ -37,6 +37,7 @@ import { CircleCITransformation } from '@/plugins/register/circleci'; import { ArgoCDTransformation } from '@/plugins/register/argocd'; import { GhCopilotTransformation } from '@/plugins/register/gh-copilot'; import { AsanaTransformation } from '@/plugins/register/asana'; +import { ClickUpTransformation } from '@/plugins/register/clickup'; import { DOC_URL } from '@/release'; import { operator } from '@/utils'; @@ -207,6 +208,15 @@ export const ScopeConfigForm = ({ /> )} + {plugin === 'clickup' && ( + + )} + {plugin === 'azuredevops' && ( + + + + diff --git a/config-ui/src/plugins/register/clickup/config.tsx b/config-ui/src/plugins/register/clickup/config.tsx new file mode 100644 index 00000000000..c98dcd00bf0 --- /dev/null +++ b/config-ui/src/plugins/register/clickup/config.tsx @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const ClickUpConfig: IPluginConfig = { + plugin: 'clickup', + name: 'ClickUp', + icon: ({ color }) => , + isBeta: true, + // Grouped with the other "C" connectors (right after CircleCI). + sort: 5, + connection: { + docLink: 'https://clickup.com/api/', + initialValues: { + endpoint: 'https://api.clickup.com/api/v2/', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'ClickUp REST API base URL. Keep the default unless self-hosted/proxied.', + }, + { + key: 'token', + label: 'API Token', + subLabel: + 'Your ClickUp personal API token (ClickUp → Settings → Apps → API Token, starts with "pk_"). Sent verbatim in the Authorization header.', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (6000).', + defaultValue: 6000, + }, + ], + }, + dataScope: { + title: 'Boards', + searchPlaceholder: 'Search folders...', + millerColumn: { + // Workspace -> Space -> Folder (the selectable board). 3 columns. + columnCount: 3, + firstColumnTitle: 'Workspace / Space', + }, + }, + scopeConfig: { + entities: ['TICKET'], + transformation: { + sprintNamePattern: '', + storyPointField: '', + defaultIssueType: '', + issueTypeRequirement: '', + issueTypeBug: '', + issueTypeIncident: '', + issueStatusTodo: [], + issueStatusInProgress: [], + issueStatusDone: [], + }, + }, +}; diff --git a/config-ui/src/plugins/register/clickup/index.ts b/config-ui/src/plugins/register/clickup/index.ts new file mode 100644 index 00000000000..5f16858cbe4 --- /dev/null +++ b/config-ui/src/plugins/register/clickup/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/clickup/transformation.tsx b/config-ui/src/plugins/register/clickup/transformation.tsx new file mode 100644 index 00000000000..f54efcd3737 --- /dev/null +++ b/config-ui/src/plugins/register/clickup/transformation.tsx @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { CaretRightOutlined } from '@ant-design/icons'; +import { theme, Collapse, Form, Input, Select } from 'antd'; + +interface Props { + entities: string[]; + connectionId: ID; + transformation: any; + setTransformation: React.Dispatch>; +} + +export const ClickUpTransformation = ({ transformation, setTransformation }: Props) => { + const { token } = theme.useToken(); + + const panelStyle: React.CSSProperties = { + marginBottom: 24, + background: token.colorFillAlter, + borderRadius: token.borderRadiusLG, + border: 'none', + }; + + const set = (patch: Record) => setTransformation({ ...transformation, ...patch }); + + return ( + } + style={{ background: token.colorBgContainer }} + size="large" + items={[ + { + key: 'TICKET', + label: 'Issue Tracking', + style: panelStyle, + children: ( + <> +

+ Tell DevLake how the lists and tasks in this ClickUp folder map onto DevLake's domain model (boards, + sprints, issues) so sprint velocity and DORA metrics are computed correctly. +

+ + + set({ sprintNamePattern: e.target.value })} + /> + + + + set({ storyPointField: e.target.value })} + /> + + + + set({ issueTypeRequirement: e.target.value })} + /> + + + set({ issueTypeBug: e.target.value })} + /> + + + set({ issueTypeIncident: e.target.value })} + /> + + +

+ Status mapping (optional). ClickUp statuses are auto-mapped by their type (open/unstarted → To Do, + custom → In Progress, done/closed → Done). List raw status names below to override. +

+ + set({ issueStatusInProgress: value })} + /> + + + set({ bugListPattern: e.target.value })} + /> + + + set({ incidentListPattern: e.target.value })} + /> + +

Status mapping (optional). ClickUp statuses are auto-mapped by their type (open/unstarted → To Do, custom → In Progress, done/closed → Done). List raw status names below to override.