From f4e9cb444c0dc8b5204cbb4609690937a6c594e9 Mon Sep 17 00:00:00 2001 From: bujjibabukatta Date: Sat, 13 Jun 2026 12:30:05 +0530 Subject: [PATCH 01/10] feat(webhook): allow closing incidents via request body instead of URL path --- backend/plugins/webhook/api/issues.go | 65 +++++++++++++++++++++++++++ backend/plugins/webhook/impl/impl.go | 9 ++++ 2 files changed, 74 insertions(+) diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index 49ea72b51f1..42ba29c2d32 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -234,6 +234,43 @@ func CloseIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro return closeIssue(input, err, connection) } +// CloseIssueByBodyRequest is the body for the body-based close endpoint +type CloseIssueByBodyRequest struct { + IssueKey string `mapstructure:"issueKey" validate:"required,max=255"` + ResolutionDate *time.Time `mapstructure:"resolutionDate"` + OriginalStatus string `mapstructure:"originalStatus"` +} + +// CloseIssueByBody +// @Summary close an issue (body-based) +// @Description Close an incident by passing issueKey in the request body. +// @Description Use this when the client (e.g. Kibana) cannot construct a dynamic URL. +// @Tags plugins/webhook +// @Param connectionId path int true "connection ID" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/{connectionId}/issue/close [POST] +func CloseIssueByBody(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + // Inject issueKey into input.Params so closeIssue() can read it + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} + // CloseIssueByName // @Summary set issue's status to DONE // @Description set issue's status to DONE @@ -247,6 +284,34 @@ func CloseIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput err := connectionHelper.FirstByName(connection, input.Params) return closeIssue(input, err, connection) } + +// CloseIssueByBodyByName +// @Summary close an issue by connection name (body-based) +// @Description Close an incident using connection name + issueKey in request body. +// @Tags plugins/webhook +// @Param connectionName path string true "connection name" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/by-name/{connectionName}/issue/close [POST] +func CloseIssueByBodyByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.FirstByName(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *models.WebhookConnection) (*plugin.ApiResourceOutput, errors.Error) { if err != nil { diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index 9a67683584e..daf07b53526 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -99,6 +99,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/:connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + "connections/:connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, ":connectionId/deployments": { "POST": api.PostDeployments, }, @@ -111,6 +114,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler ":connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + ":connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, "connections/by-name/:connectionName": { "GET": api.GetConnectionByName, "PATCH": api.PatchConnectionByName, @@ -128,6 +134,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/by-name/:connectionName/issue/:issueKey/close": { "POST": api.CloseIssueByName, }, + "connections/by-name/:connectionName/issue/close": { + "POST": api.CloseIssueByBodyByName, + }, "projects/:projectName/deployments": { "POST": api.PostDeploymentsByProjectName, }, From 9200befb33483f36e078a4c246f8f9f17940e4fd Mon Sep 17 00:00:00 2001 From: bujjibabukatta Date: Sun, 14 Jun 2026 12:47:09 +0530 Subject: [PATCH 02/10] feat(webhook): allow closing incidents via request body instead of url path --- backend/plugins/webhook/api/issues.go | 67 +++++++++++++-------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index 42ba29c2d32..5692c53e2d3 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -236,9 +236,9 @@ func CloseIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro // CloseIssueByBodyRequest is the body for the body-based close endpoint type CloseIssueByBodyRequest struct { - IssueKey string `mapstructure:"issueKey" validate:"required,max=255"` - ResolutionDate *time.Time `mapstructure:"resolutionDate"` - OriginalStatus string `mapstructure:"originalStatus"` + IssueKey string `mapstructure:"issueKey" validate:"required,max=255"` + ResolutionDate *time.Time `mapstructure:"resolutionDate"` + OriginalStatus string `mapstructure:"originalStatus"` } // CloseIssueByBody @@ -253,22 +253,21 @@ type CloseIssueByBodyRequest struct { // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/webhook/connections/{connectionId}/issue/close [POST] func CloseIssueByBody(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.WebhookConnection{} - err := connectionHelper.First(connection, input.Params) - if err != nil { - return nil, err - } - request := &CloseIssueByBodyRequest{} - if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { - return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil - } - vld = validator.New() - if err2 := errors.Convert(vld.Struct(request)); err2 != nil { - return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil - } - // Inject issueKey into input.Params so closeIssue() can read it - input.Params["issueKey"] = request.IssueKey - return closeIssue(input, err, connection) + connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) } // CloseIssueByName @@ -296,21 +295,21 @@ func CloseIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/webhook/connections/by-name/{connectionName}/issue/close [POST] func CloseIssueByBodyByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.WebhookConnection{} - err := connectionHelper.FirstByName(connection, input.Params) - if err != nil { - return nil, err - } - request := &CloseIssueByBodyRequest{} - if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { - return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil - } - vld = validator.New() - if err2 := errors.Convert(vld.Struct(request)); err2 != nil { - return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil - } - input.Params["issueKey"] = request.IssueKey - return closeIssue(input, err, connection) + connection := &models.WebhookConnection{} + err := connectionHelper.FirstByName(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) } func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *models.WebhookConnection) (*plugin.ApiResourceOutput, errors.Error) { From 52defc2712f06582661397fc59439d807e21ca67 Mon Sep 17 00:00:00 2001 From: Jarek <74616204+jaroslawgajewski@users.noreply.github.com> Date: Tue, 9 Jun 2026 03:30:19 +0200 Subject: [PATCH 03/10] feat(gh-copilot): close Copilot metrics parity gaps (#8889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gh-copilot): close API gaps for per-user metrics, teams, CLI, code review, and PR fields Add missing GitHub Copilot Metrics API fields to achieve full API parity: Enterprise/Org metrics: - CLI active user counts and CLI breakdown (sessions, requests, tokens) - Code review user counts (daily/weekly/monthly × active/passive) - Chat panel mode breakdown (agent/ask/custom/edit/plan/unknown) - Expanded PR metrics (merged, merge time, suggestions, Copilot impact) Per-user metrics: - used_cli, used_copilot_code_review_active/passive boolean flags - CLI breakdown per user (sessions, requests, tokens) User-team mapping (new): - New collector/extractor for user-teams-1-day endpoint - Enables team-level metrics via JOIN with per-user tables Seat assignments: - Team assignment fields (assigning_team id/name/slug) - User detail fields (name, email) Includes migration script 20260527 and comprehensive docs in COPILOT_METRICS_GAPS.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * feat(copilot-metrics): remove outdated metrics gaps documentation and implement new user-team mapping and metrics enhancements Signed-off-by: Jarek --------- Signed-off-by: Jarek Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ..._tool_copilot_enterprise_daily_metrics.csv | 6 +- .../snapshot_tables/_tool_copilot_seats.csv | 6 +- .../gh-copilot/models/enterprise_metrics.go | 48 +++++- .../20260527_add_copilot_metrics_gaps.go | 153 ++++++++++++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/gh-copilot/models/models.go | 2 + .../plugins/gh-copilot/models/models_test.go | 1 + backend/plugins/gh-copilot/models/seat.go | 5 + .../plugins/gh-copilot/models/user_metrics.go | 14 +- .../plugins/gh-copilot/models/user_team.go | 45 ++++++ .../tasks/enterprise_metrics_extractor.go | 100 ++++++++++-- .../gh-copilot/tasks/metrics_extractor.go | 46 ++++++ .../gh-copilot/tasks/org_metrics_collector.go | 1 + backend/plugins/gh-copilot/tasks/register.go | 2 + .../gh-copilot/tasks/seat_extractor.go | 7 + backend/plugins/gh-copilot/tasks/subtasks.go | 17 ++ .../tasks/user_metrics_extractor.go | 41 +++-- .../gh-copilot/tasks/user_teams_collector.go | 133 +++++++++++++++ .../gh-copilot/tasks/user_teams_extractor.go | 93 +++++++++++ 19 files changed, 684 insertions(+), 37 deletions(-) create mode 100644 backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go create mode 100644 backend/plugins/gh-copilot/models/user_team.go create mode 100644 backend/plugins/gh-copilot/tasks/user_teams_collector.go create mode 100644 backend/plugins/gh-copilot/tasks/user_teams_extractor.go diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv index 57e3f362712..7a74a5cc81d 100644 --- a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv @@ -1,3 +1,3 @@ -connection_id,scope_id,day,enterprise_id,daily_active_users,weekly_active_users,monthly_active_users,monthly_active_chat_users,monthly_active_agent_users,pr_total_reviewed,pr_total_created,pr_total_created_by_copilot,pr_total_reviewed_by_copilot,user_initiated_interaction_count,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum -1,octodemo,2025-09-01T00:00:00.000+00:00,,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,octodemo,2025-09-02T00:00:00.000+00:00,,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +connection_id,scope_id,day,enterprise_id,daily_active_users,weekly_active_users,monthly_active_users,monthly_active_chat_users,monthly_active_agent_users,daily_active_cli_users,daily_active_copilot_code_review_users,daily_passive_copilot_code_review_users,weekly_active_copilot_code_review_users,weekly_passive_copilot_code_review_users,monthly_active_copilot_code_review_users,monthly_passive_copilot_code_review_users,chat_panel_agent_mode,chat_panel_ask_mode,chat_panel_custom_mode,chat_panel_edit_mode,chat_panel_plan_mode,chat_panel_unknown_mode,pr_total_reviewed,pr_total_created,pr_total_merged,pr_median_minutes_to_merge,pr_total_suggestions,pr_total_applied_suggestions,pr_total_created_by_copilot,pr_total_reviewed_by_copilot,pr_total_merged_created_by_copilot,pr_total_merged_reviewed_by_copilot,pr_median_min_to_merge_copilot_authored,pr_median_min_to_merge_copilot_reviewed,pr_total_copilot_suggestions,pr_total_copilot_applied_suggestions,user_initiated_interaction_count,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum,cli_session_count,cli_request_count,cli_prompt_count,cli_output_token_sum,cli_prompt_token_sum +1,octodemo,2025-09-01T00:00:00.000+00:00,,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,octodemo,2025-09-02T00:00:00.000+00:00,,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv index 87b28f6b8fb..291a98ca607 100644 --- a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv @@ -1,3 +1,3 @@ -connection_id,organization,user_login,user_id,plan_type,created_at,last_activity_at,last_activity_editor,last_authenticated_at,pending_cancellation_date,updated_at -1,octodemo,nathos,4215,enterprise,2023-08-28T23:50:42.000+00:00,2025-11-06T16:12:15.000+00:00,copilot_pr_review,2025-12-04T15:53:22.000+00:00,,2024-02-01T00:00:00.000+00:00 -1,octodemo,octocat,1,enterprise,2024-01-10T10:11:12.000+00:00,,vscode/1.0.0/copilot-chat/0.1.0,,,2024-02-02T00:00:00.000+00:00 +connection_id,organization,user_login,user_id,user_name,user_email,plan_type,assigning_team_id,assigning_team_name,assigning_team_slug,created_at,last_activity_at,last_activity_editor,last_authenticated_at,pending_cancellation_date,updated_at +1,octodemo,nathos,4215,,,enterprise,0,,,2023-08-28T23:50:42.000+00:00,2025-11-06T16:12:15.000+00:00,copilot_pr_review,2025-12-04T15:53:22.000+00:00,,2024-02-01T00:00:00.000+00:00 +1,octodemo,octocat,1,,,enterprise,0,,,2024-01-10T10:11:12.000+00:00,,vscode/1.0.0/copilot-chat/0.1.0,,,2024-02-02T00:00:00.000+00:00 diff --git a/backend/plugins/gh-copilot/models/enterprise_metrics.go b/backend/plugins/gh-copilot/models/enterprise_metrics.go index 07663aa6dd5..967e3ecd326 100644 --- a/backend/plugins/gh-copilot/models/enterprise_metrics.go +++ b/backend/plugins/gh-copilot/models/enterprise_metrics.go @@ -44,6 +44,15 @@ type CopilotCodeMetrics struct { LocDeletedSum int `json:"locDeletedSum"` } +// CopilotCliMetrics contains CLI usage breakdown metrics. +type CopilotCliMetrics struct { + CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"` + CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"` + CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"` + CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"` + CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"` +} + // GhCopilotEnterpriseDailyMetrics captures daily enterprise-level aggregate Copilot metrics. type GhCopilotEnterpriseDailyMetrics struct { ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` @@ -57,12 +66,43 @@ type GhCopilotEnterpriseDailyMetrics struct { MonthlyActiveChatUsers int `json:"monthlyActiveChatUsers"` MonthlyActiveAgentUsers int `json:"monthlyActiveAgentUsers"` - PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` - PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` - PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` - PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + // CLI active users + DailyActiveCliUsers int `json:"dailyActiveCliUsers" gorm:"comment:Daily active CLI users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"dailyActiveCopilotCodeReviewUsers"` + DailyPassiveCopilotCodeReviewUsers int `json:"dailyPassiveCopilotCodeReviewUsers"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weeklyActiveCopilotCodeReviewUsers"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weeklyPassiveCopilotCodeReviewUsers"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthlyActiveCopilotCodeReviewUsers"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthlyPassiveCopilotCodeReviewUsers"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chatPanelAgentMode" gorm:"comment:Chat panel agent mode interactions"` + ChatPanelAskMode int `json:"chatPanelAskMode" gorm:"comment:Chat panel ask mode interactions"` + ChatPanelCustomMode int `json:"chatPanelCustomMode" gorm:"comment:Chat panel custom mode interactions"` + ChatPanelEditMode int `json:"chatPanelEditMode" gorm:"comment:Chat panel edit mode interactions"` + ChatPanelPlanMode int `json:"chatPanelPlanMode" gorm:"comment:Chat panel plan mode interactions"` + ChatPanelUnknownMode int `json:"chatPanelUnknownMode" gorm:"comment:Chat panel unknown mode interactions"` + + // Pull request metrics (expanded) + PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` + PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` + PRTotalMerged int `json:"prTotalMerged" gorm:"comment:Total PRs merged"` + PRMedianMinutesToMerge float64 `json:"prMedianMinutesToMerge" gorm:"comment:Median minutes to merge PRs"` + PRTotalSuggestions int `json:"prTotalSuggestions" gorm:"comment:Total PR review suggestions"` + PRTotalAppliedSuggestions int `json:"prTotalAppliedSuggestions" gorm:"comment:Total applied PR suggestions"` + PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` + PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + PRTotalMergedCreatedByCopilot int `json:"prTotalMergedCreatedByCopilot" gorm:"comment:Merged PRs created by Copilot"` + PRTotalMergedReviewedByCopilot int `json:"prTotalMergedReviewedByCopilot" gorm:"comment:Merged PRs reviewed by Copilot"` + PRMedianMinToMergeCopilotAuthored float64 `json:"prMedianMinToMergeCopilotAuthored" gorm:"comment:Median min to merge Copilot-authored PRs"` + PRMedianMinToMergeCopilotReviewed float64 `json:"prMedianMinToMergeCopilotReviewed" gorm:"comment:Median min to merge Copilot-reviewed PRs"` + PRTotalCopilotSuggestions int `json:"prTotalCopilotSuggestions" gorm:"comment:Total Copilot review suggestions"` + PRTotalCopilotAppliedSuggestions int `json:"prTotalCopilotAppliedSuggestions" gorm:"comment:Total Copilot applied suggestions"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go new file mode 100644 index 00000000000..f676b0e4bee --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCopilotMetricsGaps struct{} + +// --- Enterprise daily metrics: new columns --- + +type enterpriseDailyMetrics20260527 struct { + // CLI + DailyActiveCliUsers int + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int + DailyPassiveCopilotCodeReviewUsers int + WeeklyActiveCopilotCodeReviewUsers int + WeeklyPassiveCopilotCodeReviewUsers int + MonthlyActiveCopilotCodeReviewUsers int + MonthlyPassiveCopilotCodeReviewUsers int + + // Chat panel mode breakdown + ChatPanelAgentMode int + ChatPanelAskMode int + ChatPanelCustomMode int + ChatPanelEditMode int + ChatPanelPlanMode int + ChatPanelUnknownMode int + + // Expanded PR metrics + PRTotalMerged int + PRMedianMinutesToMerge float64 + PRTotalSuggestions int + PRTotalAppliedSuggestions int + PRTotalMergedCreatedByCopilot int + PRTotalMergedReviewedByCopilot int + PRMedianMinToMergeCopilotAuthored float64 + PRMedianMinToMergeCopilotReviewed float64 + PRTotalCopilotSuggestions int + PRTotalCopilotAppliedSuggestions int + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (enterpriseDailyMetrics20260527) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +// --- User daily metrics: new columns --- + +type userDailyMetrics20260527 struct { + UsedCli bool + UsedCopilotCodeReviewActive bool + UsedCopilotCodeReviewPassive bool + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (userDailyMetrics20260527) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +// --- Seat: new columns --- + +type seat20260527 struct { + UserName string `gorm:"type:varchar(255)"` + UserEmail string `gorm:"type:varchar(255)"` + AssigningTeamId int64 + AssigningTeamName string `gorm:"type:varchar(255)"` + AssigningTeamSlug string `gorm:"type:varchar(255)"` +} + +func (seat20260527) TableName() string { + return "_tool_copilot_seats" +} + +// --- User-teams: new table --- + +type userTeam20260527 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + + UserLogin string `gorm:"type:varchar(255);index"` + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + TeamSlug string `gorm:"type:varchar(255)"` + + archived.NoPKModel +} + +func (userTeam20260527) TableName() string { + return "_tool_copilot_user_teams" +} + +func (script *addCopilotMetricsGaps) Up(basicRes context.BasicRes) errors.Error { + // Add new columns to existing tables + if err := migrationhelper.AutoMigrateTables(basicRes, + &enterpriseDailyMetrics20260527{}, + &userDailyMetrics20260527{}, + &seat20260527{}, + ); err != nil { + return err + } + + // Create new user-teams table + return migrationhelper.AutoMigrateTables(basicRes, + &userTeam20260527{}, + ) +} + +func (*addCopilotMetricsGaps) Version() uint64 { + return 20260527000000 +} + +func (*addCopilotMetricsGaps) Name() string { + return "Add Copilot metrics gaps: CLI, code review, chat modes, PR expansion, user-teams" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go index a9c1a770bfa..399735695e0 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/register.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -30,5 +30,6 @@ func All() []plugin.MigrationScript { new(migrateToUsageMetricsV2), new(addPRFieldsToEnterpriseMetrics), new(addOrganizationIdToUserMetrics), + new(addCopilotMetricsGaps), } } diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go index f223c821827..5143ce5f8b7 100644 --- a/backend/plugins/gh-copilot/models/models.go +++ b/backend/plugins/gh-copilot/models/models.go @@ -45,5 +45,7 @@ func GetTablesInfo() []dal.Tabler { &GhCopilotUserMetricsByModelFeature{}, // Seat assignments &GhCopilotSeat{}, + // User-team mappings + &GhCopilotUserTeam{}, } } diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go index 8c61d222079..ef5b3eff6f5 100644 --- a/backend/plugins/gh-copilot/models/models_test.go +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -40,6 +40,7 @@ func TestGetTablesInfo(t *testing.T) { (&GhCopilotUserMetricsByLanguageModel{}).TableName(): false, (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, (&GhCopilotSeat{}).TableName(): false, + (&GhCopilotUserTeam{}).TableName(): false, } if len(tables) != len(expected) { diff --git a/backend/plugins/gh-copilot/models/seat.go b/backend/plugins/gh-copilot/models/seat.go index 85ebf177ae4..d65c80f2e30 100644 --- a/backend/plugins/gh-copilot/models/seat.go +++ b/backend/plugins/gh-copilot/models/seat.go @@ -29,7 +29,12 @@ type GhCopilotSeat struct { Organization string `gorm:"primaryKey;type:varchar(255)"` UserLogin string `gorm:"primaryKey;type:varchar(255)"` UserId int64 `gorm:"index"` + UserName string `gorm:"type:varchar(255)" json:"userName"` + UserEmail string `gorm:"type:varchar(255)" json:"userEmail"` PlanType string `gorm:"type:varchar(32)"` + AssigningTeamId int64 `json:"assigningTeamId" gorm:"comment:Team that assigned the seat"` + AssigningTeamName string `json:"assigningTeamName" gorm:"type:varchar(255)"` + AssigningTeamSlug string `json:"assigningTeamSlug" gorm:"type:varchar(255)"` CreatedAt time.Time LastActivityAt *time.Time LastActivityEditor string diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index 1f17acad80a..18e9134c226 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -30,13 +30,17 @@ type GhCopilotUserDailyMetrics struct { Day time.Time `gorm:"primaryKey;type:date" json:"day"` UserId int64 `gorm:"primaryKey" json:"userId"` - OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` - EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` - UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` - UsedAgent bool `json:"usedAgent"` - UsedChat bool `json:"usedChat"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + UsedAgent bool `json:"usedAgent"` + UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/user_team.go b/backend/plugins/gh-copilot/models/user_team.go new file mode 100644 index 00000000000..d04d55ac56f --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_team.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserTeam maps users to teams per day from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +type GhCopilotUserTeam struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + TeamId int64 `gorm:"primaryKey" json:"teamId"` + + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + TeamSlug string `json:"teamSlug" gorm:"type:varchar(255)"` + + common.NoPKModel +} + +func (GhCopilotUserTeam) TableName() string { + return "_tool_copilot_user_teams" +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go index 8686b8cc415..e98a3c4f0e5 100644 --- a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go @@ -30,13 +30,31 @@ import ( // --- Enterprise report JSON structures --- type enterpriseDayTotal struct { - Day string `json:"day"` - EnterpriseId string `json:"enterprise_id"` - DailyActiveUsers int `json:"daily_active_users"` - WeeklyActiveUsers int `json:"weekly_active_users"` - MonthlyActiveUsers int `json:"monthly_active_users"` - MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` - MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + Day string `json:"day"` + EnterpriseId string `json:"enterprise_id"` + DailyActiveUsers int `json:"daily_active_users"` + WeeklyActiveUsers int `json:"weekly_active_users"` + MonthlyActiveUsers int `json:"monthly_active_users"` + MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` + MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + DailyActiveCliUsers int `json:"daily_active_cli_users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"daily_active_copilot_code_review_users"` + DailyPassiveCopilotCodeReviewUsers int `json:"daily_passive_copilot_code_review_users"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weekly_active_copilot_code_review_users"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weekly_passive_copilot_code_review_users"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthly_active_copilot_code_review_users"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthly_passive_copilot_code_review_users"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chat_panel_agent_mode"` + ChatPanelAskMode int `json:"chat_panel_ask_mode"` + ChatPanelCustomMode int `json:"chat_panel_custom_mode"` + ChatPanelEditMode int `json:"chat_panel_edit_mode"` + ChatPanelPlanMode int `json:"chat_panel_plan_mode"` + ChatPanelUnknownMode int `json:"chat_panel_unknown_mode"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` CodeGenerationActivityCount int `json:"code_generation_activity_count"` CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` @@ -49,6 +67,7 @@ type enterpriseDayTotal struct { TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` PullRequests *pullRequestStats `json:"pull_requests"` } @@ -97,10 +116,32 @@ type totalsByLangModel struct { } type pullRequestStats struct { - TotalReviewed int `json:"total_reviewed"` - TotalCreated int `json:"total_created"` - TotalCreatedByCopilot int `json:"total_created_by_copilot"` - TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalReviewed int `json:"total_reviewed"` + TotalCreated int `json:"total_created"` + TotalMerged int `json:"total_merged"` + MedianMinutesToMerge float64 `json:"median_minutes_to_merge"` + TotalSuggestions int `json:"total_suggestions"` + TotalAppliedSuggestions int `json:"total_applied_suggestions"` + TotalCreatedByCopilot int `json:"total_created_by_copilot"` + TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalMergedCreatedByCopilot int `json:"total_merged_created_by_copilot"` + TotalMergedReviewedByCopilot int `json:"total_merged_reviewed_by_copilot"` + MedianMinToMergeCopilotAuthored float64 `json:"median_minutes_to_merge_copilot_authored"` + MedianMinToMergeCopilotReviewed float64 `json:"median_minutes_to_merge_copilot_reviewed"` + TotalCopilotSuggestions int `json:"total_copilot_suggestions"` + TotalCopilotAppliedSuggestions int `json:"total_copilot_applied_suggestions"` +} + +type totalsByCli struct { + SessionCount int `json:"session_count"` + RequestCount int `json:"request_count"` + PromptCount int `json:"prompt_count"` + TokenUsage *cliTokens `json:"token_usage"` +} + +type cliTokens struct { + OutputTokensSum int `json:"output_tokens_sum"` + PromptTokensSum int `json:"prompt_tokens_sum"` } type totalsByModelFeature struct { @@ -167,6 +208,22 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -177,11 +234,32 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/metrics_extractor.go b/backend/plugins/gh-copilot/tasks/metrics_extractor.go index 4d635c1723e..d89eababde6 100644 --- a/backend/plugins/gh-copilot/tasks/metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/metrics_extractor.go @@ -38,12 +38,21 @@ type copilotSeatResponse struct { LastActivityAt *string `json:"last_activity_at"` LastActivityEditor string `json:"last_activity_editor"` Assignee copilotAssignee `json:"assignee"` + AssigningTeam *copilotTeam `json:"assigning_team"` } type copilotAssignee struct { Login string `json:"login"` Id int64 `json:"id"` Type string `json:"type"` + Name string `json:"name"` + Email string `json:"email"` +} + +type copilotTeam struct { + Id int64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` } // ExtractOrgMetrics parses org report data from the new report download API. @@ -100,6 +109,22 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -110,11 +135,32 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go index 8f651c4821f..c3a8b5e4409 100644 --- a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go @@ -20,6 +20,7 @@ package tasks import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "time" diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go index ee1dcc797fc..3c7e5b1eeb9 100644 --- a/backend/plugins/gh-copilot/tasks/register.go +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -27,10 +27,12 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { CollectCopilotSeatAssignmentsMeta, CollectEnterpriseMetricsMeta, CollectUserMetricsMeta, + CollectUserTeamsMeta, // Extractors ExtractSeatsMeta, ExtractOrgMetricsMeta, ExtractEnterpriseMetricsMeta, ExtractUserMetricsMeta, + ExtractUserTeamsMeta, } } diff --git a/backend/plugins/gh-copilot/tasks/seat_extractor.go b/backend/plugins/gh-copilot/tasks/seat_extractor.go index 48abc3c0ce1..1a1b6b13518 100644 --- a/backend/plugins/gh-copilot/tasks/seat_extractor.go +++ b/backend/plugins/gh-copilot/tasks/seat_extractor.go @@ -96,6 +96,8 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { Organization: connection.Organization, UserLogin: seat.Assignee.Login, UserId: seat.Assignee.Id, + UserName: seat.Assignee.Name, + UserEmail: seat.Assignee.Email, PlanType: seat.PlanType, CreatedAt: createdAt, LastActivityAt: lastAct, @@ -104,6 +106,11 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { PendingCancellationDate: pendingCancel, UpdatedAt: updatedAt, } + if seat.AssigningTeam != nil { + toolSeat.AssigningTeamId = seat.AssigningTeam.Id + toolSeat.AssigningTeamName = seat.AssigningTeam.Name + toolSeat.AssigningTeamSlug = seat.AssigningTeam.Slug + } return []interface{}{toolSeat}, nil }, diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go index 24a2c95f1c5..61ed5799525 100644 --- a/backend/plugins/gh-copilot/tasks/subtasks.go +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -53,6 +53,14 @@ var CollectUserMetricsMeta = plugin.SubTaskMeta{ Description: "Collect GitHub Copilot enterprise user-level usage metrics reports", } +var CollectUserTeamsMeta = plugin.SubTaskMeta{ + Name: "collectUserTeams", + EntryPoint: CollectUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot user-team mappings from user-teams-1-day report", +} + var ExtractOrgMetricsMeta = plugin.SubTaskMeta{ Name: "extractOrgMetrics", EntryPoint: ExtractOrgMetrics, @@ -88,3 +96,12 @@ var ExtractUserMetricsMeta = plugin.SubTaskMeta{ Description: "Extract Copilot user metrics into tool-layer tables", Dependencies: []*plugin.SubTaskMeta{&CollectUserMetricsMeta}, } + +var ExtractUserTeamsMeta = plugin.SubTaskMeta{ + Name: "extractUserTeams", + EntryPoint: ExtractUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot user-team mappings into tool-layer table", + Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go index 96f5570f758..72992194063 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -46,11 +46,15 @@ type userDailyReport struct { LocDeletedSum int `json:"loc_deleted_sum"` UsedAgent bool `json:"used_agent"` UsedChat bool `json:"used_chat"` + UsedCli bool `json:"used_cli"` + UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` + UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` TotalsByIde []userTotalsByIde `json:"totals_by_ide"` TotalsByFeature []totalsByFeature `json:"totals_by_feature"` TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` } type userTotalsByIde struct { @@ -106,16 +110,19 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { var results []interface{} // Main user daily metrics - results = append(results, &models.GhCopilotUserDailyMetrics{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Day: day, - UserId: u.UserId, - OrganizationId: u.OrganizationId, - EnterpriseId: u.EnterpriseId, - UserLogin: u.UserLogin, - UsedAgent: u.UsedAgent, - UsedChat: u.UsedChat, + userMetrics := &models.GhCopilotUserDailyMetrics{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + OrganizationId: u.OrganizationId, + EnterpriseId: u.EnterpriseId, + UserLogin: u.UserLogin, + UsedAgent: u.UsedAgent, + UsedChat: u.UsedChat, + UsedCli: u.UsedCli, + UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, + UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, CodeGenerationActivityCount: u.CodeGenerationActivityCount, @@ -125,7 +132,19 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocAddedSum: u.LocAddedSum, LocDeletedSum: u.LocDeletedSum, }, - }) + } + if u.TotalsByCli != nil { + userMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: u.TotalsByCli.SessionCount, + CliRequestCount: u.TotalsByCli.RequestCount, + CliPromptCount: u.TotalsByCli.PromptCount, + } + if u.TotalsByCli.TokenUsage != nil { + userMetrics.CopilotCliMetrics.CliOutputTokenSum = u.TotalsByCli.TokenUsage.OutputTokensSum + userMetrics.CopilotCliMetrics.CliPromptTokenSum = u.TotalsByCli.TokenUsage.PromptTokensSum + } + } + results = append(results, userMetrics) // User by IDE for _, ide := range u.TotalsByIde { diff --git a/backend/plugins/gh-copilot/tasks/user_teams_collector.go b/backend/plugins/gh-copilot/tasks/user_teams_collector.go new file mode 100644 index 00000000000..2ae0200d2ef --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_collector.go @@ -0,0 +1,133 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawUserTeamsTable = "copilot_user_teams" + +// CollectUserTeams collects user-team mapping data from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +func CollectUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + var urlTemplate string + + if connection.HasEnterprise() { + urlTemplate = fmt.Sprintf("enterprises/%s/copilot/metrics/reports/user-teams-1-day", connection.Enterprise) + } else if connection.Organization != "" { + urlTemplate = fmt.Sprintf("orgs/%s/copilot/metrics/reports/user-teams-1-day", connection.Organization) + } else { + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue + } + // User-teams reports are JSONL format + records, parseErr := parseJSONL(reportBody) + if parseErr != nil { + return nil, parseErr + } + results = append(results, records...) + } + return results, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_teams_extractor.go b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go new file mode 100644 index 00000000000..72a3de8abe9 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// userTeamRecord represents a single line from the user-teams-1-day JSONL report. +type userTeamRecord struct { + Day string `json:"day"` + UserId int64 `json:"user_id"` + UserLogin string `json:"user_login"` + OrganizationId string `json:"organization_id"` + EnterpriseId string `json:"enterprise_id"` + TeamId int64 `json:"team_id"` + Slug string `json:"slug"` +} + +// ExtractUserTeams parses user-team JSONL records into the GhCopilotUserTeam model. +func ExtractUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var rec userTeamRecord + if err := errors.Convert(json.Unmarshal(row.Data, &rec)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", rec.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in user-teams report") + } + + return []interface{}{ + &models.GhCopilotUserTeam{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: rec.UserId, + TeamId: rec.TeamId, + UserLogin: rec.UserLogin, + OrganizationId: rec.OrganizationId, + EnterpriseId: rec.EnterpriseId, + TeamSlug: rec.Slug, + }, + }, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} From 018a817a6a8778b82574f137acc19d695367cef3 Mon Sep 17 00:00:00 2001 From: Volodymyr Zahorniak <7808206+zahorniak@users.noreply.github.com> Date: Tue, 9 Jun 2026 05:33:13 +0300 Subject: [PATCH 04/10] fix(plugin-circleci): populate workflow id for unfinished-job collection (#8907) (#8912) * refactor(plugin-circleci): extract unfinished-jobs input clauses into a helper Signed-off-by: Volodymyr Zahorniak * fix(plugin-circleci): populate workflow id for unfinished-job collection (#8907) The collectJobs 'unfinished details' collector built its URL from '/v2/workflow/{{ .Input.Id }}/job' but its iterator selected 'DISTINCT workflow_id' into a models.CircleciJob, leaving .Id empty and producing '/v2/workflow//job' (HTTP 500) whenever a job was running/queued/on_hold. Alias the projection to 'workflow_id AS id' so .Id carries the workflow id, mirroring the new-records collector. Adds an e2e regression test. Signed-off-by: Volodymyr Zahorniak --------- Signed-off-by: Volodymyr Zahorniak Co-authored-by: Klesh Wong --- .../circleci/e2e/job_collector_test.go | 82 +++++++++++++++++++ .../plugins/circleci/tasks/job_collector.go | 22 +++-- 2 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 backend/plugins/circleci/e2e/job_collector_test.go diff --git a/backend/plugins/circleci/e2e/job_collector_test.go b/backend/plugins/circleci/e2e/job_collector_test.go new file mode 100644 index 00000000000..ed2f014217f --- /dev/null +++ b/backend/plugins/circleci/e2e/job_collector_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "reflect" + "sort" + "testing" + + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/circleci/impl" + "github.com/apache/incubator-devlake/plugins/circleci/models" + "github.com/apache/incubator-devlake/plugins/circleci/tasks" + "github.com/stretchr/testify/assert" +) + +// TestCircleciUnfinishedJobsInputIterator is a regression test for +// https://github.com/apache/devlake/issues/8907. The "collect unfinished job +// details" collector builds its URL from "/v2/workflow/{{ .Input.Id }}/job" while +// scanning rows into a models.CircleciJob. Its input query must therefore expose the +// workflow id in the row's Id field; a bare "DISTINCT workflow_id" left Id empty and +// produced "/v2/workflow//job" (HTTP 500). This test runs the production query +// (tasks.UnfinishedJobsInputClauses) through the real iterator and asserts each +// yielded row's Id is the workflow id, that results are DISTINCT, and that the +// status/connection filters hold. +func TestCircleciUnfinishedJobsInputIterator(t *testing.T) { + var circleci impl.Circleci + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) + + const projectSlug = "github/test/repo" + dataflowTester.FlushTabler(&models.CircleciJob{}) + + seed := []models.CircleciJob{ + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-1", ProjectSlug: projectSlug, Status: "on_hold"}, + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-2", ProjectSlug: projectSlug, Status: "running"}, // same workflow -> DISTINCT + {ConnectionId: 1, WorkflowId: "wf-queued", Id: "job-3", ProjectSlug: projectSlug, Status: "queued"}, + {ConnectionId: 1, WorkflowId: "wf-success", Id: "job-4", ProjectSlug: projectSlug, Status: "success"}, // terminal -> excluded + {ConnectionId: 2, WorkflowId: "wf-otherconn", Id: "job-5", ProjectSlug: projectSlug, Status: "on_hold"}, // other connection -> excluded + } + for i := range seed { + assert.Nil(t, dataflowTester.Dal.Create(&seed[i])) + } + + cursor, err := dataflowTester.Dal.Cursor(tasks.UnfinishedJobsInputClauses(1, projectSlug)...) + assert.Nil(t, err) + iter, err := api.NewDalCursorIterator(dataflowTester.Dal, cursor, reflect.TypeOf(models.CircleciJob{})) + assert.Nil(t, err) + defer iter.Close() + + var ids []string + for iter.HasNext() { + item, err := iter.Fetch() + assert.Nil(t, err) + job := item.(*models.CircleciJob) + ids = append(ids, job.Id) + } + sort.Strings(ids) + + // Distinct workflow ids for connection 1's non-terminal jobs, with Id populated + // (the URL template reads .Input.Id). wf-success (terminal) and wf-otherconn + // (connection 2) are excluded. + assert.Equal(t, []string{"wf-onhold", "wf-queued"}, ids) + for _, id := range ids { + assert.NotEmpty(t, id, "Input.Id must be the workflow id, not empty (#8907)") + } +} diff --git a/backend/plugins/circleci/tasks/job_collector.go b/backend/plugins/circleci/tasks/job_collector.go index fd1d78286bf..00fd234524f 100644 --- a/backend/plugins/circleci/tasks/job_collector.go +++ b/backend/plugins/circleci/tasks/job_collector.go @@ -41,6 +41,20 @@ var CollectJobsMeta = plugin.SubTaskMeta{ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, } +// UnfinishedJobsInputClauses returns the DAL clauses that select the workflows whose +// jobs are still in a non-terminal status and therefore need their job details +// recollected by the CollectJobs "unfinished details" collector. +func UnfinishedJobsInputClauses(connectionId uint64, projectSlug string) []dal.Clause { + return []dal.Clause{ + dal.Select("DISTINCT workflow_id AS id"), // #8907: alias to id so {{ .Input.Id }} resolves when scanned into CircleciJob + dal.From(&models.CircleciJob{}), + dal.Where( + "connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", + connectionId, projectSlug, + ), + } +} + func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) logger := taskCtx.GetLogger() @@ -94,14 +108,8 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { AfterResponse: ignoreDeletedBuilds, }, BuildInputIterator: func() (api.Iterator, errors.Error) { - clauses := []dal.Clause{ - dal.Select("DISTINCT workflow_id"), // Only need to recollect jobs for a workflow once - dal.From(&models.CircleciJob{}), - dal.Where("connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", data.Options.ConnectionId, data.Options.ProjectSlug), - } - db := taskCtx.GetDal() - cursor, err := db.Cursor(clauses...) + cursor, err := db.Cursor(UnfinishedJobsInputClauses(data.Options.ConnectionId, data.Options.ProjectSlug)...) if err != nil { return nil, err } From bb5cdb7684c47d19b6cc1d6661606ff2ba85c997 Mon Sep 17 00:00:00 2001 From: Kevin Sanjula Date: Fri, 12 Jun 2026 10:33:43 +0530 Subject: [PATCH 05/10] Fix/graphql client token refresh (#8791) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(github-graphql): prevent panic in graphql rate limit polling goroutine Replace panic in GraphqlAsyncClient rate-limit polling goroutine with graceful error handling. Previously, any error while fetching rate limit (e.g., transient network issues or 401 responses) would trigger a panic inside a background goroutine, crashing the entire DevLake process. Now, errors are logged and the client retries in the next cycle while retaining the last known rate limit. Design decisions: - Avoid panic in background goroutines: rate-limit polling is non-critical and should not bring down the entire pipeline. - Use last known rateRemaining on runtime failures instead of resetting or blocking, ensuring continued progress with eventual consistency. - Retry via existing polling mechanism instead of immediate retry to prevent tight retry loops and unnecessary API pressure. - Introduce a default fallback (5000) only for initial rate-limit fetch failures, since no prior state exists at startup. - Separate handling of initial vs runtime failures: - Initial failure → fallback to default (5000) - Runtime failure → retain previous value Fixes #8788 (bug 1) * fix(github-graphql): reuse ApiClient transport for GraphQL to enable token refresh Replace oauth2.StaticTokenSource-based HTTP client with the underlying http.Client from ApiAsyncClient. Previously, the GraphQL client constructed its own HTTP client using StaticTokenSource, which froze the access token at task start time. This caused GitHub App installation tokens (which expire after ~1 hour) to become invalid during long-running pipelines, leading to persistent 401 errors. Now, the GraphQL client reuses apiClient.GetClient(), which is already configured with RefreshRoundTripper and TokenProvider. This enables automatic token refresh on 401 responses, aligning GraphQL behavior with the REST client. Design decisions: - Reuse transport layer instead of duplicating authentication logic to ensure consistency across REST and GraphQL clients. - Avoid StaticTokenSource, as it prevents token refresh and breaks long-running pipelines. - Leverage existing RefreshRoundTripper for transparent token rotation without modifying GraphQL query logic. - Keep protocol-specific logic (GraphQL vs REST) separate while sharing the underlying HTTP transport. This ensures GraphQL pipelines using GitHub App authentication can run beyond token expiry without failure. Fixes #8788 (bug 2) * refactor(github): extract shared authenticated http client from api client - moved token provider and refresh round tripper setup into a reusable helper - introduced CreateAuthenticatedHttpClient to centralize auth + transport logic - updated CreateApiClient to use shared http client instead of inline setup Rationale: - decouples authentication (transport layer) from REST-specific client logic - enables reuse for GraphQL client without duplicating token refresh logic - aligns architecture with separation of concerns (http transport vs api clients) * feat(github-graphql): introduce graphql client with shared auth and integrate into task flow - added CreateGraphqlClient to encapsulate graphql client construction - reused CreateAuthenticatedHttpClient from github/tasks to inject token refresh via RoundTripper - replaced manual graphql client setup in PrepareTaskData with new factory function - preserved existing rate limit handling via getRateRemaining callback - preserved query cost calculation using SetGetRateCost Technical details: - graphql client now uses http transport with TokenProvider and RefreshRoundTripper - removes dependency on oauth2 client and avoids token expiration issues - decouples graphql client from REST ApiClient by avoiding reuse of apiClient.GetClient() - maintains compatibility with github.com and enterprise graphql endpoints Note: - shared auth logic remains in github/tasks and is imported with alias to avoid package name collision - introduces cross-plugin dependency (github_graphql → github/tasks) as a pragmatic tradeoff to avoid duplication * feat(github): support static token transport for GraphQL and REST clients add StaticRoundTripper for PAT authentication and use it in the shared http client. since the same client is used by both REST and GraphQL, auth handling must distinguish between refreshable tokens and static tokens. avoid applying refresh/retry logic to PAT. ensures correct behavior across clients and prevents unnecessary retries for static auth. * feat(github-graphql): introduce hierarchical fallback for GraphQL rate limit Implement a layered fallback mechanism for GraphQL rate limiting: 1. Dynamic rate limit from provider (getRateRemaining) 2. Per-client override (WithFallbackRateLimit) 3. Config override (GRAPHQL_RATE_LIMIT) 4. Default fallback (1000) Also moved GitHub-specific fallback (5000) via WithFallbackRateLimit to the Graphql client. * feat(github-graphql): Add graphql rate limit to .env example * fix(github): Fix leaked debug statement * fix(github-graphql): reuse http.Client proxy, auth configurations Reused `http.Client` inside the apiClient returned by `CreateApiClient` method, so keeping the proxy and auth configurations the same.That also keep the centralized management of logic. * fix(helpers): fix the priority order of fallback rate limit Priority order fixed for fallback rate limit, priority order is: 1.Env variable 2.Value set with `WithFallbackRateLimit` 3.default value in the code This all works only when the `getRateRemaining` fails: hence the fallback * fix(github): StaticRoundTripper now owns token splitting and rotation for AccessToken connections Previously, connection.Token (comma-separated PATs) was injected as-is into the Authorization header, sending "Bearer tok1,tok2,tok3" instead of a single rotated token. StaticRoundTripper now splits the raw token string on comma and rotates through tokens round-robin using an atomic counter. For REST: StaticRoundTripper operates at transport level and always overwrites the Authorization header set by SetupAuthentication. SetupAuthentication is retained because conn.tokens is still required by GetTokensCount() for rate limit calculation — but its header write is superseded by StaticRoundTripper on every request. For GraphQL: SetupAuthentication is never called by the graphql client, so StaticRoundTripper is the only auth mechanism on this path — without this fix, GraphQL requests were sent with the full unsplit token string. * refactor(github-graphql): Downgrade fetch failure logs from Warn to Info * fix(helper): use inline func type for GraphqlClientOption to avoid mock cycle Replace exported GraphqlClientOption type with inline func(*GraphqlAsyncClient) in CreateAsyncGraphqlClient signature. The named type caused mockery to generate a mock file (GraphqlClientOption.go) that created an import cycle in tests. * style(github): fix linting * fix(github): token rotation start from index * fix(helper): prevent graphql deadlock when rate limit fetch keeps failing --------- Co-authored-by: Klesh Wong --- .../pluginhelper/api/graphql_async_client.go | 70 +++++++++++++-- backend/plugins/github/tasks/api_client.go | 39 +------- backend/plugins/github/tasks/http_client.go | 88 +++++++++++++++++++ backend/plugins/github/token/round_tripper.go | 36 ++++++++ backend/plugins/github_graphql/impl/impl.go | 51 ++--------- .../github_graphql/tasks/graphql_client.go | 66 ++++++++++++++ env.example | 2 + 7 files changed, 265 insertions(+), 87 deletions(-) create mode 100644 backend/plugins/github/tasks/http_client.go create mode 100644 backend/plugins/github_graphql/tasks/graphql_client.go diff --git a/backend/helpers/pluginhelper/api/graphql_async_client.go b/backend/helpers/pluginhelper/api/graphql_async_client.go index 32dac66a9eb..61a1c9fa3ca 100644 --- a/backend/helpers/pluginhelper/api/graphql_async_client.go +++ b/backend/helpers/pluginhelper/api/graphql_async_client.go @@ -20,12 +20,14 @@ package api import ( "context" "fmt" + "strconv" + "sync" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/core/utils" - "sync" - "time" "github.com/merico-ai/graphql" ) @@ -47,30 +49,52 @@ type GraphqlAsyncClient struct { getRateCost func(q interface{}) int } +// defaultRateLimitConst is the generic fallback rate limit for GraphQL requests. +// It is used as the initial remaining quota when dynamic rate limit +// information is unavailable from the provider. +const defaultRateLimitConst = 1000 + // CreateAsyncGraphqlClient creates a new GraphqlAsyncClient func CreateAsyncGraphqlClient( taskCtx plugin.TaskContext, graphqlClient *graphql.Client, logger log.Logger, getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), + opts ...func(*GraphqlAsyncClient), ) (*GraphqlAsyncClient, errors.Error) { ctxWithCancel, cancel := context.WithCancel(taskCtx.GetContext()) + graphqlAsyncClient := &GraphqlAsyncClient{ ctx: ctxWithCancel, cancel: cancel, client: graphqlClient, logger: logger, rateExhaustCond: sync.NewCond(&sync.Mutex{}), - rateRemaining: 0, + rateRemaining: defaultRateLimitConst, getRateRemaining: getRateRemaining, } + // apply options + for _, opt := range opts { + opt(graphqlAsyncClient) + } + + // Env config wins over everything, only if explicitly set + if rateLimit := resolveRateLimit(taskCtx, logger); rateLimit != -1 { + logger.Info("GRAPHQL_RATE_LIMIT env override applied: %d (was %d)", rateLimit, graphqlAsyncClient.rateRemaining) + graphqlAsyncClient.rateRemaining = rateLimit + } + if getRateRemaining != nil { rateRemaining, resetAt, err := getRateRemaining(taskCtx.GetContext(), graphqlClient, logger) if err != nil { - panic(err) + graphqlAsyncClient.logger.Info("failed to fetch initial graphql rate limit, fallback to default: %v", err) + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) + } else { + graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) } - graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) + } else { + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) } // load retry/timeout from configuration @@ -115,6 +139,10 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese apiClient.rateExhaustCond.Signal() } go func() { + if apiClient.getRateRemaining == nil { + return + } + nextDuring := 3 * time.Minute if resetAt != nil && resetAt.After(time.Now()) { nextDuring = time.Until(*resetAt) @@ -126,7 +154,15 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese case <-time.After(nextDuring): newRateRemaining, newResetAt, err := apiClient.getRateRemaining(apiClient.ctx, apiClient.client, apiClient.logger) if err != nil { - panic(err) + apiClient.logger.Info("failed to update graphql rate limit, will retry next cycle: %v", err) + // Floor the reused value so Signal() always fires; prevents deadlock when + // rateRemaining is 0 and the rate-limit endpoint keeps erroring (e.g. GHE). + fallback := apiClient.rateRemaining + if fallback < defaultRateLimitConst { + fallback = defaultRateLimitConst + } + apiClient.updateRateRemaining(fallback, nil) + return } apiClient.updateRateRemaining(newRateRemaining, newResetAt) } @@ -218,3 +254,25 @@ func (apiClient *GraphqlAsyncClient) Wait() { func (apiClient *GraphqlAsyncClient) Release() { apiClient.cancel() } + +// WithFallbackRateLimit sets the initial/fallback rate limit used when +// rate limit information cannot be fetched dynamically. +// This value may be overridden later by getRateRemaining. +func WithFallbackRateLimit(limit int) func(*GraphqlAsyncClient) { + return func(c *GraphqlAsyncClient) { + if limit > 0 { + c.rateRemaining = limit + } + } +} + +// resolveRateLimit returns -1 if GRAPHQL_RATE_LIMIT is not set or invalid +func resolveRateLimit(taskCtx plugin.TaskContext, logger log.Logger) int { + if v := taskCtx.GetConfig("GRAPHQL_RATE_LIMIT"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil { + return parsed + } + logger.Warn(nil, "invalid GRAPHQL_RATE_LIMIT, using default") + } + return -1 +} diff --git a/backend/plugins/github/tasks/api_client.go b/backend/plugins/github/tasks/api_client.go index 42181ff139e..c5be8a0f22b 100644 --- a/backend/plugins/github/tasks/api_client.go +++ b/backend/plugins/github/tasks/api_client.go @@ -26,7 +26,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "github.com/apache/incubator-devlake/plugins/github/token" ) func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnection) (*api.ApiAsyncClient, errors.Error) { @@ -35,40 +34,10 @@ func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnec return nil, err } - logger := taskCtx.GetLogger() - db := taskCtx.GetDal() - encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) - - // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. - var tp *token.TokenProvider - if connection.RefreshToken != "" { - tp = token.NewTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { - tp = token.NewAppInstallationTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } - if tp != nil { - // Wrap the transport - baseTransport := apiClient.GetClient().Transport - if baseTransport == nil { - baseTransport = http.DefaultTransport - } - - rt := token.NewRefreshRoundTripper(baseTransport, tp) - apiClient.GetClient().Transport = rt - logger.Info("Installed token refresh round tripper for connection %d (authMethod=%s)", - connection.ID, connection.AuthMethod) - } - - // Persist the freshly minted token so the DB has a correctly encrypted value. - // PrepareApiClient (called by NewApiClientFromConnection) mints the token - // in-memory but does not persist it; without this, the DB may contain a stale - // or corrupted token that breaks GET /connections. - if connection.AuthMethod == models.AppKey && connection.Token != "" { - if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { - logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) - } else { - logger.Info("Persisted initial token for connection %d", connection.ID) - } + // inject the shared auth layer + _, err = CreateAuthenticatedHttpClient(taskCtx, connection, apiClient.GetClient()) + if err != nil { + return nil, err } // create rate limit calculator diff --git a/backend/plugins/github/tasks/http_client.go b/backend/plugins/github/tasks/http_client.go new file mode 100644 index 00000000000..33ef6a3df6b --- /dev/null +++ b/backend/plugins/github/tasks/http_client.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/github/token" +) + +func CreateAuthenticatedHttpClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + baseClient *http.Client, +) (*http.Client, errors.Error) { + + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) + + if baseClient == nil { + baseClient = &http.Client{} + } + + // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. + var tp *token.TokenProvider + if connection.RefreshToken != "" { + tp = token.NewTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { + tp = token.NewAppInstallationTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } + + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + if tp != nil { + baseClient.Transport = token.NewRefreshRoundTripper(baseTransport, tp) + logger.Info( + "Installed token refresh round tripper for connection %d (authMethod=%s)", + connection.ID, + connection.AuthMethod, + ) + + } else if connection.Token != "" { + baseClient.Transport = token.NewStaticRoundTripper( + baseTransport, + connection.Token, + ) + logger.Info( + "Installed static token round tripper for connection %d", + connection.ID, + ) + } + + // Persist the freshly minted token so the DB has a correctly encrypted value. + // PrepareApiClient (called by NewApiClientFromConnection) mints the token + // in-memory but does not persist it; without this, the DB may contain a stale + // or corrupted token that breaks GET /connections. + if connection.AuthMethod == models.AppKey && connection.Token != "" { + if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { + logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) + } else { + logger.Info("Persisted initial token for connection %d", connection.ID) + } + } + + return baseClient, nil +} diff --git a/backend/plugins/github/token/round_tripper.go b/backend/plugins/github/token/round_tripper.go index 8868572dae6..8f05f838d0d 100644 --- a/backend/plugins/github/token/round_tripper.go +++ b/backend/plugins/github/token/round_tripper.go @@ -19,6 +19,8 @@ package token import ( "net/http" + "strings" + "sync/atomic" ) // RefreshRoundTripper is an HTTP transport middleware that automatically manages OAuth token refreshes. @@ -93,3 +95,37 @@ func (rt *RefreshRoundTripper) roundTripWithRetry(req *http.Request, refreshAtte return resp, nil } + +// StaticRoundTripper is an HTTP transport that injects a fixed bearer token. +// Unlike RefreshRoundTripper, it does NOT attempt refresh or retries. +type StaticRoundTripper struct { + base http.RoundTripper + tokens []string + idx atomic.Uint64 +} + +func NewStaticRoundTripper(base http.RoundTripper, rawToken string) *StaticRoundTripper { + if base == nil { + base = http.DefaultTransport + } + parts := strings.Split(rawToken, ",") + tokens := make([]string, 0, len(parts)) + for _, t := range parts { + if t = strings.TrimSpace(t); t != "" { + tokens = append(tokens, t) + } + } + if len(tokens) == 0 { + tokens = []string{rawToken} + } + return &StaticRoundTripper{base: base, tokens: tokens} +} + +func (rt *StaticRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // always overrides headers put by SetupAuthentication, to make sure the token is always injected + // Add(1)-1 yields a 0-based sequence (0, 1, 2, ...) so rotation starts at tokens[0]. + tok := rt.tokens[(rt.idx.Add(1)-1)%uint64(len(rt.tokens))] + reqClone := req.Clone(req.Context()) + reqClone.Header.Set("Authorization", "Bearer "+tok) + return rt.base.RoundTrip(reqClone) +} diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go index 3efbe10a272..f56c77644bf 100644 --- a/backend/plugins/github_graphql/impl/impl.go +++ b/backend/plugins/github_graphql/impl/impl.go @@ -20,10 +20,7 @@ package impl import ( "context" "fmt" - "net/http" - "net/url" "reflect" - "strings" "time" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -39,7 +36,6 @@ import ( "github.com/apache/incubator-devlake/plugins/github_graphql/model/migrationscripts" "github.com/apache/incubator-devlake/plugins/github_graphql/tasks" "github.com/merico-ai/graphql" - "golang.org/x/oauth2" ) // make sure interface is implemented @@ -180,46 +176,10 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return nil, err } - tokens := strings.Split(connection.Token, ",") - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: tokens[0]}, - ) - oauthContext := taskCtx.GetContext() - proxy := connection.GetProxy() - if proxy != "" { - pu, err := url.Parse(proxy) - if err != nil { - return nil, errors.Convert(err) - } - if pu.Scheme == "http" || pu.Scheme == "socks5" { - proxyClient := &http.Client{ - Transport: &http.Transport{Proxy: http.ProxyURL(pu)}, - } - oauthContext = context.WithValue( - taskCtx.GetContext(), - oauth2.HTTPClient, - proxyClient, - ) - logger.Debug("Proxy set in oauthContext to %s", proxy) - } else { - return nil, errors.BadInput.New("Unsupported scheme set in proxy") - } - } - - httpClient := oauth2.NewClient(oauthContext, src) - endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) - if err != nil { - return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) - } - - // github.com and github enterprise have different graphql endpoints - endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql - if endpoint.Hostname() != "api.github.com" { - // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql - endpoint.Path = "/api/graphql" - } - client := graphql.NewClient(endpoint.String(), httpClient) - graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + graphqlClient, err := tasks.CreateGraphqlClient( + taskCtx, + connection, + apiClient.ApiClient.GetClient(), func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) { var query GraphQueryRateLimit dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil)) @@ -230,8 +190,7 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`) } if query.RateLimit == nil { - logger.Info(`github graphql rate limit are disabled, fallback to 5000req/hour`) - return 5000, nil, nil + return 0, nil, errors.Default.New("rate limit unavailable") } logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`, query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt) diff --git a/backend/plugins/github_graphql/tasks/graphql_client.go b/backend/plugins/github_graphql/tasks/graphql_client.go new file mode 100644 index 00000000000..9c248e15cb6 --- /dev/null +++ b/backend/plugins/github_graphql/tasks/graphql_client.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/merico-ai/graphql" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +func CreateGraphqlClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + httpClient *http.Client, + getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), +) (*helper.GraphqlAsyncClient, errors.Error) { + // Build endpoint + endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) + } + // github.com and github enterprise have different graphql endpoints + if endpoint.Hostname() == "api.github.com" { + // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/graphql" + } else { + // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/api/graphql" + } + + gqlClient := graphql.NewClient(endpoint.String(), httpClient) + + return helper.CreateAsyncGraphqlClient( + taskCtx, + gqlClient, + taskCtx.GetLogger(), + getRateRemaining, + // GitHub GraphQL default fallback aligns with GitHub's standard rate limit (~5000) + helper.WithFallbackRateLimit(5000), + ) +} diff --git a/env.example b/env.example index 843df675245..6141f24978b 100755 --- a/env.example +++ b/env.example @@ -40,6 +40,8 @@ PUSH_API_ALLOWED_TABLES= NOTIFICATION_ENDPOINT= NOTIFICATION_SECRET= +# Default fallback rate limit for GraphQL clients (used if not dynamically fetched) +GRAPHQL_RATE_LIMIT= API_TIMEOUT=120s API_RETRY=3 API_REQUESTS_PER_HOUR=10000 From 69ce9f8f382671b9b6165daaa6cf46a65574c2f4 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Fri, 12 Jun 2026 16:21:22 +0300 Subject: [PATCH 06/10] fix(github): create domain accounts for non-committer authors (#8886) (#8894) * fix(github): create domain accounts for non-committer authors (#8886) ConvertAccounts sourced the domain `accounts` table FROM _tool_github_accounts, which is only populated for users we collected full profiles for (effectively, committers). Issue and PR authors who never committed were written into _tool_github_repo_accounts but never converted, so issues.creator_id and pull_requests.author_id pointed at accounts rows that didn't exist. Source ConvertAccounts FROM _tool_github_repo_accounts LEFT JOIN _tool_github_accounts instead, so every user the repo references gets a domain account, enriched with profile detail when we have it and login-only otherwise. The domain id uses the same generator the issue/PR convertors use, so the FKs line up. Also emit a repo_account for a PR's merged_by user so pull_requests.merged_by_id resolves too. The query stays MySQL/PostgreSQL-agnostic (COALESCE, no backtick quoting, parameterized via the dal) and mirrors the join already in account_org_collector.go. Adds the non-committer orphan case to the e2e fixture plus a referential- integrity assertion in TestAccountDataFlow. Verified on both MySQL and PostgreSQL. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(github): carry _raw_data provenance and guard zero account ids Review feedback on #8894: the rewritten ConvertAccounts dropped the _raw_data fields from converted accounts. Select them with COALESCE, preferring the enriched _tool_github_accounts row and falling back to the _tool_github_repo_accounts row for non-committers. The e2e fixture now carries the _raw_data columns like every other tool fixture, and the regenerated snapshot pins the pre-existing provenance for enriched accounts plus the issue-extractor provenance for the non-committer case. Also guard PR author and merged-by id generation against zero ids: an unmerged PR or a deleted user otherwise yields the domain id github:GithubAccount::0, the same orphan-FK shape this PR fixes. issue_convertor already guards its AuthorId the same way. Verified with the full github e2e suite on both MySQL and PostgreSQL. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/plugins/github/e2e/account_test.go | 33 +++++++++ .../raw_tables/_tool_github_repo_accounts.csv | 33 ++++----- .../github/e2e/snapshot_tables/account.csv | 1 + .../plugins/github/tasks/account_convertor.go | 70 +++++++++++++++---- backend/plugins/github/tasks/pr_convertor.go | 11 ++- backend/plugins/github/tasks/pr_extractor.go | 10 +++ 6 files changed, 126 insertions(+), 32 deletions(-) diff --git a/backend/plugins/github/e2e/account_test.go b/backend/plugins/github/e2e/account_test.go index 817ef6fab22..7b6466aba23 100644 --- a/backend/plugins/github/e2e/account_test.go +++ b/backend/plugins/github/e2e/account_test.go @@ -20,11 +20,15 @@ package e2e import ( "testing" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/github/impl" "github.com/apache/incubator-devlake/plugins/github/models" "github.com/apache/incubator-devlake/plugins/github/tasks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAccountDataFlow(t *testing.T) { @@ -108,4 +112,33 @@ func TestAccountDataFlow(t *testing.T) { "_raw_data_remark", }, ) + + // Referential-integrity invariant (#8886): every account the repo references in + // _tool_github_repo_accounts must have a domain `accounts` row, so issues.creator_id / + // pull_requests.author_id / merged_by_id never point at a missing account. We generate + // the domain id with the SAME generator the issue/PR convertors use, so this is a + // faithful proxy for the FK join the issue reported as broken. It also fails loudly if a + // future change shrinks ConvertAccounts' coverage or diverges the id generation. + accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) + var repoAccounts []models.GithubRepoAccount + require.NoError(t, dataflowTester.Dal.All(&repoAccounts, + dal.Where("repo_github_id = ? AND connection_id = ? AND account_id > 0", + taskData.Options.GithubId, taskData.Options.ConnectionId), + )) + require.NotEmpty(t, repoAccounts, "fixture must reference at least one account") + sawOrphanCase := false + for _, ra := range repoAccounts { + if ra.Login == "milichev" { + sawOrphanCase = true // the non-committer author from the issue repro + } + domainId := accountIdGen.Generate(taskData.Options.ConnectionId, ra.AccountId) + count, err := dataflowTester.Dal.Count( + dal.From(&crossdomain.Account{}), + dal.Where("id = ?", domainId), + ) + require.NoError(t, err) + assert.Equalf(t, int64(1), count, + "orphan FK: repo account %q (id=%d) has no domain accounts row %q", ra.Login, ra.AccountId, domainId) + } + assert.True(t, sawOrphanCase, "fixture should include the non-committer orphan case (milichev)") } diff --git a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv index 8b86664428f..7aa42a4b0a9 100644 --- a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv +++ b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv @@ -1,16 +1,17 @@ -connection_id,account_id,repo_github_id,login -1,21979,134018330,appleboy -1,964542,134018330,sarathsp06 -1,1052632,134018330,runner-mei -1,3794113,134018330,shanhuhai5739 -1,3971390,134018330,ppmoon -1,7496278,134018330,panjf2000 -1,8518239,134018330,gitter-badger -1,11763614,2,Moonlight-Zhao -1,12420699,2,shanghai-Jerry -1,14950473,2,zqkgo -1,22429695,2,codecov[bot] -1,24841832,2,rikewang -1,31087327,2,chensanle -1,32893410,2,zhangyuanxue -1,38849208,2,king526 \ No newline at end of file +connection_id,account_id,repo_github_id,login,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,21979,134018330,appleboy,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, +1,964542,134018330,sarathsp06,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,1, +1,1052632,134018330,runner-mei,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +1,3794113,134018330,shanhuhai5739,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, +1,3971390,134018330,ppmoon,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, +1,7496278,134018330,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,5, +1,8518239,134018330,gitter-badger,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,9, +1,145564,134018330,milichev,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, +1,11763614,2,Moonlight-Zhao,,,0, +1,12420699,2,shanghai-Jerry,,,0, +1,14950473,2,zqkgo,,,0, +1,22429695,2,codecov[bot],,,0, +1,24841832,2,rikewang,,,0, +1,31087327,2,chensanle,,,0, +1,32893410,2,zhangyuanxue,,,0, +1,38849208,2,king526,,,0, diff --git a/backend/plugins/github/e2e/snapshot_tables/account.csv b/backend/plugins/github/e2e/snapshot_tables/account.csv index 1092e7a5296..c3022a4a296 100644 --- a/backend/plugins/github/e2e/snapshot_tables/account.csv +++ b/backend/plugins/github/e2e/snapshot_tables/account.csv @@ -1,5 +1,6 @@ id,email,full_name,user_name,avatar_url,organization,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark github:GithubAccount:1:1052632,runner.mei@,runner,runner-mei,https://avatars.githubusercontent.com/u/1052632?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +github:GithubAccount:1:145564,,,milichev,,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, github:GithubAccount:1:21979,appleboy.tw@gmail.com,Bo-Yi Wu,appleboy,https://avatars.githubusercontent.com/u/21979?v=4,"COSCUP,nodejs-tw,moztw,h5bp,CodeIgniter-TW,drone,Getmore,golangtw,laravel-taiwan,go-xorm,gin-gonic,PHPConf-TW,Mediatek-Cloud,SJFinder,go-gitea,laradock,gin-contrib,tagfans,maintainers,go-training,go-ggz,the-benchmarker,golang-queue","{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, github:GithubAccount:1:3794113,shanhu5739@gmail.com,Derek,shanhuhai5739,https://avatars.githubusercontent.com/u/3794113?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, github:GithubAccount:1:3971390,cnliuyunpeng@gmail.com,ppmoon,ppmoon,https://avatars.githubusercontent.com/u/3971390?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go index e86780d2746..3d4db97d155 100644 --- a/backend/plugins/github/tasks/account_convertor.go +++ b/backend/plugins/github/tasks/account_convertor.go @@ -22,6 +22,7 @@ import ( "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -30,6 +31,19 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +// repoAccountForConvert is the row projected by ConvertAccounts' query: every +// account referenced by the repo (from _tool_github_repo_accounts), enriched +// with profile detail from _tool_github_accounts when it was collected. The +// embedded NoPKModel carries the RawDataOrigin across to the domain row. +type repoAccountForConvert struct { + Id int + Login string + Name string + Email string + AvatarUrl string + common.NoPKModel +} + func init() { RegisterSubtaskMeta(&ConvertAccountsMeta) } @@ -38,12 +52,12 @@ var ConvertAccountsMeta = plugin.SubTaskMeta{ Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, - Description: "Convert tool layer table github_accounts into domain layer table accounts", + Description: "Convert every account referenced by the repo (tool layer repo_accounts, enriched by github_accounts) into domain layer table accounts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, DependencyTables: []string{ - models.GithubAccount{}.TableName(), // cursor - models.GithubRepoAccount{}.TableName(), // cursor - models.GithubAccountOrg{}.TableName()}, // account id gen + models.GithubRepoAccount{}.TableName(), // cursor (every user referenced by the repo) + models.GithubAccount{}.TableName(), // left-join enrichment (profile detail, optional) + models.GithubAccountOrg{}.TableName()}, // org pluck ProductTables: []string{crossdomain.Account{}.TableName()}, } @@ -53,7 +67,7 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubAccount]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[repoAccountForConvert]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: taskCtx, Table: RAW_ACCOUNT_TABLE, @@ -62,29 +76,57 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { Name: data.Options.Name, }, }, + // Source every account referenced by this repo from _tool_github_repo_accounts + // (which the issue/PR/commit extractors populate for any author, assignee, or + // merged-by user), and LEFT JOIN _tool_github_accounts for profile detail when it + // was collected. This guarantees a domain `accounts` row for every CreatorId / + // AuthorId the other convertors emit, instead of only for users who committed. + // Raw-data provenance follows the same rule as the profile fields: the enriched + // _tool_github_accounts row when we collected one, the repo_accounts row otherwise. + // Note the consequence: fallback-provenance rows carry a _raw_data_table other than + // _raw_github_api_accounts, so the batch-save divider's full-sync delete-then-reinsert + // (keyed on this converter's raw table) never deletes them; they are reconciled by + // upsert only. Scope deletion still covers them via _raw_data_params. + // SQL is kept DB-agnostic (no backtick quoting, COALESCE not IFNULL) so it runs on + // both MySQL and PostgreSQL. Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ - dal.Select("_tool_github_accounts.*"), - dal.From(&models.GithubAccount{}), + dal.Select(`_tool_github_repo_accounts.account_id AS id, + _tool_github_repo_accounts.login AS login, + COALESCE(ga.name, '') AS name, + COALESCE(ga.email, '') AS email, + COALESCE(ga.avatar_url, '') AS avatar_url, + COALESCE(ga._raw_data_params, _tool_github_repo_accounts._raw_data_params) AS _raw_data_params, + COALESCE(ga._raw_data_table, _tool_github_repo_accounts._raw_data_table) AS _raw_data_table, + COALESCE(ga._raw_data_id, _tool_github_repo_accounts._raw_data_id) AS _raw_data_id, + COALESCE(ga._raw_data_remark, _tool_github_repo_accounts._raw_data_remark) AS _raw_data_remark`), + dal.From(&models.GithubRepoAccount{}), + dal.Join(`left join _tool_github_accounts ga on ( + ga.connection_id = _tool_github_repo_accounts.connection_id + AND ga.id = _tool_github_repo_accounts.account_id + )`), dal.Where( - "repo_github_id = ? and _tool_github_accounts.connection_id=?", + `_tool_github_repo_accounts.repo_github_id = ? + AND _tool_github_repo_accounts.connection_id = ? + AND _tool_github_repo_accounts.account_id > 0`, data.Options.GithubId, data.Options.ConnectionId, ), - dal.Join(`left join _tool_github_repo_accounts gra on ( - _tool_github_accounts.connection_id = gra.connection_id - AND _tool_github_accounts.id = gra.account_id - )`), } if stateManager.IsIncremental() { since := stateManager.GetSince() if since != nil { - clauses = append(clauses, dal.Where("_tool_github_accounts.updated_at >= ?", since)) + // Incremental cursor intentionally tracks _tool_github_repo_accounts.updated_at + // (repo membership), not _tool_github_accounts.updated_at (profile freshness): + // account-detail re-enrichment is reconciled on the next full sync. Do not switch + // this back to _tool_github_accounts — that is what left issue/PR-only authors + // orphaned (#8886). + clauses = append(clauses, dal.Where("_tool_github_repo_accounts.updated_at >= ?", since)) } } return db.Cursor(clauses...) }, - Convert: func(githubUser *models.GithubAccount) ([]interface{}, errors.Error) { + Convert: func(githubUser *repoAccountForConvert) ([]interface{}, errors.Error) { // query related orgs var orgs []string err := db.Pluck(`org_login`, &orgs, diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 48a40cc4e67..c4e27a6b9bb 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -87,7 +87,6 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { OriginalStatus: pr.State, Title: pr.Title, Url: pr.Url, - AuthorId: accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId), AuthorName: pr.AuthorName, Description: pr.Body, CreatedDate: pr.GithubCreatedAt, @@ -104,9 +103,17 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { Additions: pr.Additions, Deletions: pr.Deletions, MergedByName: pr.MergedByName, - MergedById: accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById), IsDraft: pr.IsDraft, } + // Generate account ids only for real users (#8886): a zero AuthorId (deleted + // user) or zero MergedById (unmerged PR) would otherwise produce an id like + // github:GithubAccount:1:0 that no accounts row can ever match. + if pr.AuthorId != 0 { + domainPr.AuthorId = accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId) + } + if pr.MergedById != 0 { + domainPr.MergedById = accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById) + } if pr.State == "open" || pr.State == "OPEN" { domainPr.Status = code.OPEN } else if pr.State == "MERGED" || (pr.State == "closed" && (pr.Merged || pr.MergedAt != nil)) { diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go index 802846f301c..9c3f1ca8fc9 100644 --- a/backend/plugins/github/tasks/pr_extractor.go +++ b/backend/plugins/github/tasks/pr_extractor.go @@ -151,6 +151,16 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { githubPr.AuthorName = githubUser.Login githubPr.AuthorId = githubUser.AccountId } + // Emit a repo_account for the merged-by user too, so pull_requests.merged_by_id + // resolves to a domain account instead of an orphan FK (ConvertAccounts sources + // every referenced user from _tool_github_repo_accounts). + if body.MergedBy != nil { + mergedByUser, err := convertAccount(body.MergedBy, data.Options.GithubId, data.Options.ConnectionId) + if err != nil { + return nil, err + } + results = append(results, mergedByUser) + } for _, label := range body.Labels { results = append(results, &models.GithubPrLabel{ ConnectionId: data.Options.ConnectionId, From a64d21f14c9afa3228c28155206e171a0ac6b078 Mon Sep 17 00:00:00 2001 From: bujjibabukatta Date: Fri, 12 Jun 2026 19:01:18 +0530 Subject: [PATCH 07/10] fix(webhook): set resolution_date and lead_time_minutes on incident close (#8919) --- backend/plugins/webhook/api/issues.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index 5692c53e2d3..ae6919b7bfa 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -329,6 +329,16 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo } domainIssue.Status = ticket.DONE domainIssue.OriginalStatus = `` + now := time.Now() + if domainIssue.ResolutionDate == nil { + domainIssue.ResolutionDate = &now + } + if domainIssue.LeadTimeMinutes == nil || *domainIssue.LeadTimeMinutes == 0 { + if domainIssue.CreatedDate != nil { + temp := uint(domainIssue.ResolutionDate.Sub(*domainIssue.CreatedDate).Minutes()) + domainIssue.LeadTimeMinutes = &temp + } + } // save err = tx.Update(domainIssue) if err != nil { @@ -342,6 +352,15 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo if err == nil { domainIncident.Status = ticket.DONE domainIncident.OriginalStatus = `` + if domainIncident.ResolutionDate == nil { + domainIncident.ResolutionDate = &now + } + if domainIncident.LeadTimeMinutes == nil || *domainIncident.LeadTimeMinutes == 0 { + if domainIncident.CreatedDate != nil { + temp := uint(domainIncident.ResolutionDate.Sub(*domainIncident.CreatedDate).Minutes()) + domainIncident.LeadTimeMinutes = &temp + } + } // save err = tx.Update(domainIncident) if err != nil { From 34125fb350896a2a172afe7952087a741a1c8d9f Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Date: Fri, 12 Jun 2026 23:48:47 +1000 Subject: [PATCH 08/10] feat: add Linear (linear.app) data source plugin (#8900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(linear): add tool-layer models and init migration Add the Linear plugin's tool-layer data models (connection, team scope, scope config, account, issue, comment, issue label, workflow state, cycle, issue history) and the initial schema migration with archived snapshots. The connection authenticates with a personal API key passed verbatim in the Authorization header (Linear uses no Bearer prefix). Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): add plugin skeleton, connection API and GraphQL client Wire the Linear plugin entry point and implement all required plugin interfaces (meta, init, task, api, model, source, migration, blueprint v200, closeable). Add connection/scope/scope-config CRUD via the data-source helper, a test-connection endpoint that runs a GraphQL viewer query, and a rate-limited async GraphQL client that injects the API key via a bare Authorization header. SubTaskMetas is intentionally empty; collectors are added per entity in following commits. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect, extract and convert users to accounts Add the users GraphQL collector (paginated), extractor to _tool_linear_accounts, and convertor to the domain crossdomain.Account table, wired as the first three subtasks. Includes an e2e dataflow test with raw fixtures and verified snapshots. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect and extract workflow states Add the team-scoped workflow states GraphQL collector and extractor into _tool_linear_workflow_states. These states (backlog/unstarted/started/ completed/canceled) drive deterministic issue status mapping. Includes an e2e test covering all five state types. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect, extract and convert issues Add the team-scoped issues GraphQL collector (incremental via updatedAt ordering, inline labels), extractor to _tool_linear_issues and _tool_linear_issue_labels, and convertor to domain ticket.Issue and ticket.BoardIssue. Status maps deterministically from Linear's WorkflowState.type (backlog/unstarted->TODO, started->IN_PROGRESS, completed/canceled->DONE); priority maps to its label; lead time falls back to resolution minus creation. Includes an e2e test spanning all state types, unassigned issues, issues without a cycle, and multi-label issues. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect, extract and convert issue comments Add a per-issue comments GraphQL collector (driven by an input iterator over collected issues, with pagination), an extractor that recovers the owning issue id from the raw input column, and a convertor to domain ticket.IssueComment. Includes an e2e dataflow test. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): convert issue labels to domain layer Add the convertor from _tool_linear_issue_labels (populated inline by the issue extractor) into the domain ticket.IssueLabel table. Includes an e2e test covering issues with multiple labels and with none. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect cycles and convert to sprints Add the team-scoped cycles GraphQL collector and extractor, plus convertors producing domain ticket.Sprint and ticket.BoardSprint (status derived from completedAt), and ticket.SprintIssue linking issues to their cycle. Includes an e2e dataflow test covering closed/active cycles and issues with/without a cycle. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): collect issue history and convert to changelogs Add a per-issue history GraphQL collector (input iterator over issues, with pagination), an extractor capturing state transitions including state types, and a convertor to domain ticket.IssueChangelogs with mapped from/to status values. Lead time is already derived from the issue's native startedAt/completedAt. Includes an e2e test of a full backlog->started->completed lifecycle. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * test(linear): add blueprint v200 scope generation tests Cover makeScopesV200: a team scope with the ticket entity produces the expected domain board scope id, and a scope without the ticket entity produces none. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * docs(linear): add plugin README Document the Linear plugin: supported entities, tool/domain mapping tables, deterministic status mapping, priority/type/lead-time handling, API-key auth, connection/scope/pipeline setup examples, rate limiting, and the roadmap (OAuth, label-based type mapping, config-ui integration). Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): guard lead-time fallback against resolution before creation A resolution timestamp (completedAt/canceledAt) earlier than createdAt — from clock skew or migrated/imported issues — produced a negative duration that, cast to uint, yields platform-dependent garbage (0 on arm64, ~1.8e19 on amd64). Skip the fallback unless the resolution is after creation so lead time stays unset instead. Adds an isolated e2e dataflow test with a fixture whose canceledAt precedes createdAt, asserting lead_time_minutes is empty. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): map Linear triage state type to TODO The WorkflowState.type 'triage' (the inbox state issues land in before being accepted) previously fell through to OTHER, contradicting the documented total mapping and silently mislabeling triage issues. Map it to TODO; keep OTHER as the fallback for genuinely unrecognized types so unexpected API values surface. Adds a unit test covering every documented state type plus triage and an unknown value. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * refactor(linear): remove unused GraphqlInlineAccount struct The struct was documented as the shared inline-user shape but was never referenced; each collector declares its own inline user struct. Removing it avoids misleading a maintainer into editing a type nothing reads. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * perf(linear): raise issue collector page size to 100 Every other Linear collector uses a page size of 100; issues used 50, which doubled the number of issue-page round-trips and the iterator size that drives the per-issue comment/history collectors. Linear permits first: 250. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): populate issue assignee/creator names and issue_assignees The issue convertor set only assignee_id/creator_id, leaving the denormalized assignee_name/creator_name columns blank and writing no issue_assignees rows, so dashboards reading those columns or joining through issue_assignees showed blank names. Preload account display names (matching the account convertor's displayName-then-name rule) and emit an IssueAssignee per assigned issue. The issue dataflow test now loads accounts before conversion and asserts the names plus issue_assignees; the lead-time test flushes accounts to stay order-independent on the shared test DB. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): clear stale sprint_issues when issues leave their cycle Sprint membership is derived from each issue's cycle_id, and the batch divider only deletes outdated rows when it produces at least one row of the type. When every issue is moved out of its cycle the convertor emits nothing, so the divider never fires and prior sprint_issues rows linger, leaving issues shown in sprints they no longer belong to. Delete the team's sprint_issues up front so the result is correct regardless of how many issues remain in a cycle. Adds a two-run e2e test that empties every issue's cycle and asserts sprint_issues is empty afterward. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): derive issue lead time from state-transition history The LinearIssue.LeadTimeMinutes field was never populated, so lead time always fell back to the coarse createdAt -> resolutionDate span. Derive it instead from the recorded history: the span from an issue's first transition into an in-progress state to its first transition into a done state thereafter (active cycle time), which is the value that genuinely requires history. ConvertIssues still seeds the fallback; ConvertIssueHistory now overrides it when the transitions exist, and issues lacking them keep the fallback. Adds an e2e test asserting issue-1 (started 05-02, completed 05-03) resolves to 1440 minutes from history rather than its 2880-minute created->resolved span. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): add remote-scopes endpoints to enumerate teams Other first-party ticket plugins (jira, asana, github) expose connections/:connectionId/remote-scopes so the config UI can browse and select scopes from the API. Linear had none, forcing users to hand-craft a PUT /scopes with raw team UUIDs they had no in-product way to discover. Wire the standard DsRemoteApiProxyHelper + DsRemoteApiScopeListHelper and a lister that queries the GraphQL teams connection (flat list, cursor-paginated) through the connection's authenticated client. Adds unit tests for the response->scope-entry mapping, the pagination cursor, and the route registration. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * perf(linear): make comment and history collection incremental Both child collectors used a plain GraphqlCollector and swept every issue in the team on every run, issuing one request per issue with no since filter - tens of thousands of requests per run on a large team against Linear's ~1500 req/hour budget. Switch them to a stateful collector and restrict the driving cursor to issues updated since the last successful collection, so steady-state runs scale with the change delta rather than the whole backlog. A full sync (since == nil) still sweeps every issue. Adds a unit test for the incremental cursor-clause builder. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): filter issues server-side by updatedAt for incremental sync Incremental collection relied on the issues query returning newest-first and a client-side early-stop, but the query pinned no sort direction (Linear's orderBy is a scalar enum with no direction operand). If the server default were ascending, the early-stop would fire on the first (oldest) row and collect almost nothing. Pass a server-side IssueFilter { updatedAt: { gt: since } } instead and drop the early-stop, so correctness no longer depends on an undocumented default ordering. A full sync passes an empty filter (match all). Adds a unit test pinning the filter's JSON shape to Linear's IssueFilter input. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * refactor(linear): drop dead LeadTimeMinutes tool-layer field The _tool_linear_issues.lead_time_minutes column was never populated (the collector never requested it and no extractor set it). Now that lead time is derived into the domain ticket.Issue directly -- from state-transition history when available, otherwise the createdAt->resolutionDate fallback in the issue convertor -- the tool-layer field is pure dead weight. Remove it from the model, the init migration's archived model, the convertor, and the extractor snapshot. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(config-ui): register Linear plugin Adds the Linear data source to config-ui so it appears in the connection picker: connection form (endpoint + personal API key + proxy + rate limit), a flat Teams data-scope backed by the plugin's remote-scopes endpoint, and the Linear logo. No scope-config transformation — Linear's status mapping is deterministic. Wired into the plugin registry. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(config-ui): map Linear scope id to teamId getPluginScopeId fell through to the default (scope.id) for Linear, but a LinearTeam scope is keyed by teamId and has no id field — so the blueprint referenced an undefined scopeId and patching failed with 'LinearTeam not found'. Add a linear case returning scope.teamId. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): add Grafana dashboard Adds grafana/dashboards/Linear.json (cloned from the Asana ticket-dashboard template) so Linear ships a per-tool dashboard like every other ticket plugin. Its board picker is scoped to Linear (boards id like 'linear%'); the 13 panels (throughput, lead/cycle time, status distribution, delivery rate, sprints) read the shared domain tables. Auto-loaded via Grafana file provisioning. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): convert team scope to a domain board board_issues and sprint_issues referenced a board_id (boardIdGen over LinearTeam), but nothing ever created the ticket.Board row itself, so the domain boards table stayed empty. Board-scoped dashboards (whose board picker is 'boards where id like linear%') and any board join therefore returned no data. Add a ConvertTeams subtask that converts the team scope in _tool_linear_teams into a ticket.Board keyed identically to those references. Adds an e2e test asserting the board is produced with the matching id. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): widen issue title/url columns to avoid truncation _tool_linear_issues.title and .url were varchar(255), but Linear titles can exceed 255 chars (and the issue URL embeds a title slug), so extraction failed with 'Error 1406: Data too long for column title'. Drop the varchar limit so both are longtext, matching the domain issues.title and jira's tool summary. Adds an e2e test extracting a 300-char title without truncation. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * fix(linear): recover owning issue id for comments and history The GraphQL collector stores the query variables (which carry issueId) in the raw row's input column, but the comment and history extractors parsed it as {"Id":...} (SimpleLinearIssue.Id), so the owning issue id came out empty and the convertor joins produced zero domain comments/changelogs on real data. The e2e fixtures hand-wrote {"Id":...}, masking it. Parse issueId (with an Id fallback) and update the fixtures to the real collector shape. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * test: flush LinearIssueLabel before issue extraction in e2e tests The issue extractor writes both _tool_linear_issues and _tool_linear_issue_labels, but comment_test, cycle_test and issue_history_test only flushed LinearIssue before running ExtractIssuesMeta. On a clean database (as in CI) the table _tool_linear_issue_labels was never auto-migrated, so the extractor's DELETE on that table panicked and aborted the whole package. Flush LinearIssueLabel too, matching the other linear e2e tests and the jira plugin convention. Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * test: register linear plugin in Test_GetPluginTablesInfo The plugin count check failed in CI (actual 41 vs tested 40) because the linear plugin was not listed in table_info_test.go. Add its import and FeedIn call so every Go plugin is covered. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> * feat(linear): map issues to ticket types via label-based scope config Linear has no native issue type, so the issue convertor previously hardcoded every issue to REQUIREMENT. Add optional regex fields to the Linear scope config (issueTypeIncident/Bug/Requirement) matched against an issue's label names, with precedence INCIDENT > BUG > REQUIREMENT and a REQUIREMENT default when nothing matches. This lets DORA classify Linear bugs as incidents for change-failure-rate / time-to-restore. Also fix two gaps that made scope configs unusable for Linear: - thread the scope's ScopeConfigId into the pipeline task options so the convertor can load the config at runtime - register the standard scope-config/:scopeConfigId/projects route that every other plugin serves (config-ui 404'd on the scope config page) Tested: new e2e TestLinearIssueIncidentMapping (Bug label -> INCIDENT, other issues REQUIREMENT) plus a blueprint plan test asserting ScopeConfigId is passed through; full plugins/linear suite green. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> --------- Signed-off-by: Eduardo Rodrigues <2961314+eduardoarantes@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- backend/plugins/linear/README.md | 115 ++ backend/plugins/linear/api/blueprint_v200.go | 99 ++ .../plugins/linear/api/blueprint_v200_test.go | 122 ++ backend/plugins/linear/api/connection_api.go | 177 +++ backend/plugins/linear/api/init.go | 51 + backend/plugins/linear/api/remote_api.go | 137 ++ backend/plugins/linear/api/remote_api_test.go | 66 + backend/plugins/linear/api/scope_api.go | 105 ++ .../plugins/linear/api/scope_config_api.go | 106 ++ backend/plugins/linear/e2e/account_test.go | 58 + backend/plugins/linear/e2e/board_test.go | 56 + backend/plugins/linear/e2e/comment_test.go | 64 + backend/plugins/linear/e2e/cycle_test.go | 77 ++ .../linear/e2e/issue_history_leadtime_test.go | 77 ++ .../plugins/linear/e2e/issue_history_test.go | 64 + .../plugins/linear/e2e/issue_incident_test.go | 67 + .../plugins/linear/e2e/issue_leadtime_test.go | 64 + .../linear/e2e/issue_longtitle_test.go | 55 + backend/plugins/linear/e2e/issue_test.go | 77 ++ backend/plugins/linear/e2e/label_test.go | 55 + .../e2e/raw_tables/_raw_linear_accounts.csv | 4 + .../e2e/raw_tables/_raw_linear_comments.csv | 4 + .../e2e/raw_tables/_raw_linear_cycles.csv | 3 + .../raw_tables/_raw_linear_issue_history.csv | 4 + .../e2e/raw_tables/_raw_linear_issues.csv | 6 + .../_raw_linear_issues_long_title.csv | 2 + .../_raw_linear_issues_negative_leadtime.csv | 2 + .../_raw_linear_workflow_states.csv | 6 + .../snapshot_tables/_tool_linear_accounts.csv | 4 + .../snapshot_tables/_tool_linear_comments.csv | 4 + .../snapshot_tables/_tool_linear_cycles.csv | 3 + .../_tool_linear_issue_history.csv | 4 + .../_tool_linear_issue_labels.csv | 4 + .../snapshot_tables/_tool_linear_issues.csv | 6 + .../snapshot_tables/_tool_linear_teams.csv | 2 + .../_tool_linear_workflow_states.csv | 6 + .../linear/e2e/snapshot_tables/accounts.csv | 4 + .../e2e/snapshot_tables/board_issues.csv | 6 + .../e2e/snapshot_tables/board_sprints.csv | 3 + .../linear/e2e/snapshot_tables/boards.csv | 2 + .../e2e/snapshot_tables/issue_assignees.csv | 3 + .../e2e/snapshot_tables/issue_changelogs.csv | 4 + .../e2e/snapshot_tables/issue_comments.csv | 4 + .../e2e/snapshot_tables/issue_labels.csv | 4 + .../linear/e2e/snapshot_tables/issues.csv | 6 + .../issues_history_leadtime.csv | 6 + .../e2e/snapshot_tables/issues_incident.csv | 6 + .../issues_negative_leadtime.csv | 2 + .../e2e/snapshot_tables/sprint_issues.csv | 3 + .../sprint_issues_after_leaving_cycle.csv | 1 + .../linear/e2e/snapshot_tables/sprints.csv | 3 + .../plugins/linear/e2e/sprint_issue_test.go | 71 + .../plugins/linear/e2e/workflow_state_test.go | 48 + backend/plugins/linear/impl/impl.go | 233 ++++ backend/plugins/linear/impl/impl_test.go | 38 + backend/plugins/linear/linear.go | 43 + backend/plugins/linear/models/account.go | 38 + backend/plugins/linear/models/connection.go | 73 + backend/plugins/linear/models/cycle.go | 41 + backend/plugins/linear/models/issue.go | 56 + .../plugins/linear/models/issue_comment.go | 40 + .../plugins/linear/models/issue_history.go | 45 + backend/plugins/linear/models/issue_label.go | 35 + .../20260601_add_init_tables.go | 51 + .../migrationscripts/archived/models.go | 169 +++ .../models/migrationscripts/register.go | 29 + backend/plugins/linear/models/scope_config.go | 44 + backend/plugins/linear/models/team.go | 66 + .../plugins/linear/models/workflow_state.go | 39 + .../plugins/linear/tasks/account_collector.go | 98 ++ .../plugins/linear/tasks/account_convertor.go | 97 ++ .../plugins/linear/tasks/account_extractor.go | 74 + backend/plugins/linear/tasks/api_client.go | 108 ++ .../plugins/linear/tasks/board_convertor.go | 93 ++ .../plugins/linear/tasks/comment_collector.go | 148 ++ .../plugins/linear/tasks/comment_convertor.go | 97 ++ .../plugins/linear/tasks/comment_extractor.go | 78 ++ .../plugins/linear/tasks/cycle_collector.go | 102 ++ .../plugins/linear/tasks/cycle_convertor.go | 106 ++ .../plugins/linear/tasks/cycle_extractor.go | 72 + .../plugins/linear/tasks/issue_collector.go | 164 +++ .../linear/tasks/issue_collector_test.go | 41 + .../plugins/linear/tasks/issue_convertor.go | 229 ++++ .../plugins/linear/tasks/issue_extractor.go | 108 ++ .../linear/tasks/issue_history_collector.go | 137 ++ .../linear/tasks/issue_history_convertor.go | 168 +++ .../linear/tasks/issue_history_extractor.go | 85 ++ .../plugins/linear/tasks/label_convertor.go | 85 ++ backend/plugins/linear/tasks/shared.go | 86 ++ .../linear/tasks/shared_clauses_test.go | 40 + backend/plugins/linear/tasks/shared_test.go | 45 + .../linear/tasks/sprint_issue_convertor.go | 107 ++ backend/plugins/linear/tasks/task_data.go | 46 + .../linear/tasks/workflow_state_collector.go | 100 ++ .../linear/tasks/workflow_state_extractor.go | 71 + backend/plugins/table_info_test.go | 4 +- config-ui/src/plugins/register/index.ts | 2 + .../plugins/register/linear/assets/icon.svg | 19 + .../src/plugins/register/linear/config.tsx | 57 + .../src/plugins/register/linear/index.ts | 19 + config-ui/src/plugins/utils.ts | 2 + grafana/dashboards/Linear.json | 1192 +++++++++++++++++ 102 files changed, 6851 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/linear/README.md create mode 100644 backend/plugins/linear/api/blueprint_v200.go create mode 100644 backend/plugins/linear/api/blueprint_v200_test.go create mode 100644 backend/plugins/linear/api/connection_api.go create mode 100644 backend/plugins/linear/api/init.go create mode 100644 backend/plugins/linear/api/remote_api.go create mode 100644 backend/plugins/linear/api/remote_api_test.go create mode 100644 backend/plugins/linear/api/scope_api.go create mode 100644 backend/plugins/linear/api/scope_config_api.go create mode 100644 backend/plugins/linear/e2e/account_test.go create mode 100644 backend/plugins/linear/e2e/board_test.go create mode 100644 backend/plugins/linear/e2e/comment_test.go create mode 100644 backend/plugins/linear/e2e/cycle_test.go create mode 100644 backend/plugins/linear/e2e/issue_history_leadtime_test.go create mode 100644 backend/plugins/linear/e2e/issue_history_test.go create mode 100644 backend/plugins/linear/e2e/issue_incident_test.go create mode 100644 backend/plugins/linear/e2e/issue_leadtime_test.go create mode 100644 backend/plugins/linear/e2e/issue_longtitle_test.go create mode 100644 backend/plugins/linear/e2e/issue_test.go create mode 100644 backend/plugins/linear/e2e/label_test.go create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv create mode 100644 backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/accounts.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/board_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/boards.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv create mode 100644 backend/plugins/linear/e2e/snapshot_tables/sprints.csv create mode 100644 backend/plugins/linear/e2e/sprint_issue_test.go create mode 100644 backend/plugins/linear/e2e/workflow_state_test.go create mode 100644 backend/plugins/linear/impl/impl.go create mode 100644 backend/plugins/linear/impl/impl_test.go create mode 100644 backend/plugins/linear/linear.go create mode 100644 backend/plugins/linear/models/account.go create mode 100644 backend/plugins/linear/models/connection.go create mode 100644 backend/plugins/linear/models/cycle.go create mode 100644 backend/plugins/linear/models/issue.go create mode 100644 backend/plugins/linear/models/issue_comment.go create mode 100644 backend/plugins/linear/models/issue_history.go create mode 100644 backend/plugins/linear/models/issue_label.go create mode 100644 backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go create mode 100644 backend/plugins/linear/models/migrationscripts/archived/models.go create mode 100644 backend/plugins/linear/models/migrationscripts/register.go create mode 100644 backend/plugins/linear/models/scope_config.go create mode 100644 backend/plugins/linear/models/team.go create mode 100644 backend/plugins/linear/models/workflow_state.go create mode 100644 backend/plugins/linear/tasks/account_collector.go create mode 100644 backend/plugins/linear/tasks/account_convertor.go create mode 100644 backend/plugins/linear/tasks/account_extractor.go create mode 100644 backend/plugins/linear/tasks/api_client.go create mode 100644 backend/plugins/linear/tasks/board_convertor.go create mode 100644 backend/plugins/linear/tasks/comment_collector.go create mode 100644 backend/plugins/linear/tasks/comment_convertor.go create mode 100644 backend/plugins/linear/tasks/comment_extractor.go create mode 100644 backend/plugins/linear/tasks/cycle_collector.go create mode 100644 backend/plugins/linear/tasks/cycle_convertor.go create mode 100644 backend/plugins/linear/tasks/cycle_extractor.go create mode 100644 backend/plugins/linear/tasks/issue_collector.go create mode 100644 backend/plugins/linear/tasks/issue_collector_test.go create mode 100644 backend/plugins/linear/tasks/issue_convertor.go create mode 100644 backend/plugins/linear/tasks/issue_extractor.go create mode 100644 backend/plugins/linear/tasks/issue_history_collector.go create mode 100644 backend/plugins/linear/tasks/issue_history_convertor.go create mode 100644 backend/plugins/linear/tasks/issue_history_extractor.go create mode 100644 backend/plugins/linear/tasks/label_convertor.go create mode 100644 backend/plugins/linear/tasks/shared.go create mode 100644 backend/plugins/linear/tasks/shared_clauses_test.go create mode 100644 backend/plugins/linear/tasks/shared_test.go create mode 100644 backend/plugins/linear/tasks/sprint_issue_convertor.go create mode 100644 backend/plugins/linear/tasks/task_data.go create mode 100644 backend/plugins/linear/tasks/workflow_state_collector.go create mode 100644 backend/plugins/linear/tasks/workflow_state_extractor.go create mode 100644 config-ui/src/plugins/register/linear/assets/icon.svg create mode 100644 config-ui/src/plugins/register/linear/config.tsx create mode 100644 config-ui/src/plugins/register/linear/index.ts create mode 100644 grafana/dashboards/Linear.json diff --git a/backend/plugins/linear/README.md b/backend/plugins/linear/README.md new file mode 100644 index 00000000000..e52b4527176 --- /dev/null +++ b/backend/plugins/linear/README.md @@ -0,0 +1,115 @@ + + +# Linear + +## Summary + +This plugin collects data from [Linear](https://linear.app) through its +[GraphQL API](https://linear.app/developers/graphql) and maps it into DevLake's +standardized `ticket` domain, so Linear issues appear in DevLake dashboards +(throughput, lead/cycle time, sprint burndown, etc.). + +The selectable **scope** is a Linear **Team**, which maps to a domain `Board`. + +## Supported data + +| Linear entity | Tool-layer table | Domain-layer table | +|-----------------|-----------------------------------|--------------------------------------------| +| Team | `_tool_linear_teams` (scope) | `boards` | +| User | `_tool_linear_accounts` | `accounts` | +| Workflow state | `_tool_linear_workflow_states` | (drives issue status mapping) | +| Issue | `_tool_linear_issues` | `issues`, `board_issues` | +| Label | `_tool_linear_issue_labels` | `issue_labels` | +| Comment | `_tool_linear_comments` | `issue_comments` | +| Cycle | `_tool_linear_cycles` | `sprints`, `board_sprints`, `sprint_issues`| +| Issue history | `_tool_linear_issue_history` | `issue_changelogs` | + +### Field mapping highlights + +- **Status** — derived deterministically from Linear's `WorkflowState.type` + (no manual mapping needed, unlike Jira): + - `backlog`, `unstarted` → `TODO` + - `started` → `IN_PROGRESS` + - `completed`, `canceled` → `DONE` +- **Priority** — Linear's integer priority maps to a label: `0` No priority, + `1` Urgent, `2` High, `3` Medium, `4` Low. +- **Type** — Linear has no native issue type, so issues default to `REQUIREMENT`. +- **Lead time** — `completedAt − createdAt` (Linear provides `startedAt`/`completedAt` + natively; the history changelog captures every status transition). +- **Story points** — Linear's `estimate`. + +## Authentication + +The plugin uses a Linear **personal API key**, passed verbatim in the +`Authorization` header (no `Bearer` prefix). Create one under +**Settings → Security & access → Personal API keys** in Linear. + +## Configuration + +Create a connection: + +``` +curl 'http://localhost:8080/api/plugins/linear/connections' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear", + "endpoint": "https://api.linear.app/graphql", + "token": "", + "rateLimitPerHour": 1500 +}' +``` + +Add a team scope (the team id is the Linear team UUID): + +``` +curl 'http://localhost:8080/api/plugins/linear/connections//scopes' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "data": [{ "connectionId": , "teamId": "", "name": "Engineering" }] +}' +``` + +## Collecting data + +``` +curl 'http://localhost:8080/api/pipelines' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear pipeline", + "plan": [[{ + "plugin": "linear", + "options": { "connectionId": , "teamId": "" } + }]] +}' +``` + +## Rate limiting + +Linear enforces a per-API-key request budget (1,500 requests/hour) plus a +complexity budget. The collector paces requests against the configured +`rateLimitPerHour` (default 1500). Issues are collected incrementally using +`updatedAt` ordering so re-runs only fetch changes. + +## Limitations / roadmap + +- Authentication is personal API key only; OAuth2 is a planned follow-up. +- Issue type defaults to `REQUIREMENT`; label-based type mapping via the scope + config is a planned follow-up. +- config-ui integration (connection form + team picker) and the website + documentation page are planned follow-ups; for now connections and scopes are + managed via the API calls shown above. diff --git a/backend/plugins/linear/api/blueprint_v200.go b/backend/plugins/linear/api/blueprint_v200.go new file mode 100644 index 00000000000..a26a43ab521 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "linear", + subtaskMetas, + scopeConfig.Entities, + tasks.LinearOptions{ + ConnectionId: connection.ID, + TeamId: scope.TeamId, + ScopeConfigId: scope.ScopeConfigId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.TeamId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/linear/api/blueprint_v200_test.go b/backend/plugins/linear/api/blueprint_v200_test.go new file mode 100644 index 00000000000..fe0d8b05db3 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200_test.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/stretchr/testify/assert" +) + +func mockLinearPlugin(t *testing.T) { + mockMeta := mockplugin.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/linear") + mockMeta.On("Name").Return("linear").Maybe() + _ = plugin.RegisterPlugin("linear", mockMeta) +} + +func TestMakeScopesV200(t *testing.T) { + mockLinearPlugin(t) + + const connectionId uint64 = 1 + const teamId = "team-1" + const expectDomainScopeId = "linear:LinearTeam:1:team-1" + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: connectionId}, + TeamId: teamId, + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: connectionId}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) +} + +func TestMakePipelinePlanV200PassesScopeConfigId(t *testing.T) { + const scopeConfigId uint64 = 42 + subtaskMetas := []plugin.SubTaskMeta{ + {Name: "convertIssues", EnabledByDefault: true, DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}}, + } + + plan, err := makePipelinePlanV200( + subtaskMetas, + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1, ScopeConfigId: scopeConfigId}, + TeamId: "team-1", + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(plan)) + assert.Equal(t, 1, len(plan[0])) + // the scope's scopeConfigId must be threaded into the task options so the + // convertor can resolve label-based issue-type mapping at runtime. + assert.EqualValues(t, scopeConfigId, plan[0][0].Options["scopeConfigId"]) +} + +func TestMakeScopesV200WithoutTicketEntity(t *testing.T) { + mockLinearPlugin(t) + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1}, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + // no ticket entity selected => no domain board scope produced + assert.Equal(t, 0, len(scopes)) +} diff --git a/backend/plugins/linear/api/connection_api.go b/backend/plugins/linear/api/connection_api.go new file mode 100644 index 00000000000..c2c95c54d64 --- /dev/null +++ b/backend/plugins/linear/api/connection_api.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +const defaultEndpoint = "https://api.linear.app/graphql" + +type LinearTestConnResponse struct { + shared.ApiBody + Connection *models.LinearConn +} + +func testConnection(ctx context.Context, connection models.LinearConn) (*LinearTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = defaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // Linear is GraphQL-over-HTTP-POST; a minimal viewer query verifies the key. + reqBody := map[string]interface{}{"query": "{ viewer { id name } }"} + res, err := apiClient.Post("", nil, reqBody, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API key") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := LinearTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test linear connection +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param body body models.LinearConn true "json body" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.LinearConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test linear connection by ID +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.LinearConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create linear connection +// @Summary create linear connection +// @Description Create linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch linear connection +// @Summary patch linear connection +// @Description Patch linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a linear connection +// @Summary delete a linear connection +// @Description Delete a linear connection +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all linear connections +// @Summary get all linear connections +// @Description Get all linear connections +// @Tags plugins/linear +// @Success 200 {object} []models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get linear connection detail +// @Summary get linear connection detail +// @Description Get linear connection detail +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go new file mode 100644 index 00000000000..850acf54825 --- /dev/null +++ b/backend/plugins/linear/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.LinearConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.LinearConnection, models.LinearTeam, models.LinearScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.LinearConnection) models.LinearConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.LinearConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination](raProxy, listLinearRemoteScopes) +} diff --git a/backend/plugins/linear/api/remote_api.go b/backend/plugins/linear/api/remote_api.go new file mode 100644 index 00000000000..094c785016d --- /dev/null +++ b/backend/plugins/linear/api/remote_api.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearRemotePagination drives cursor-based pagination through the GraphQL +// `teams` connection when listing remote scopes for the config UI. +type LinearRemotePagination struct { + Cursor string `json:"cursor"` +} + +// linearTeamsGraphqlResponse mirrors the shape of the `teams` query response. +type linearTeamsGraphqlResponse struct { + Data struct { + Teams struct { + Nodes []struct { + Id string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + Description string `json:"description"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + } `json:"teams"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const remoteScopesPageSize = 100 + +// listLinearRemoteScopes lists Linear teams as selectable scopes. Linear teams +// are a flat list, so there are no intermediate groups. +func listLinearRemoteScopes( + _ *models.LinearConnection, + apiClient plugin.ApiClient, + _ string, + page LinearRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], + nextPage *LinearRemotePagination, + err errors.Error, +) { + after := "" + if page.Cursor != "" { + after = fmt.Sprintf(", after: %q", page.Cursor) + } + query := fmt.Sprintf( + "query { teams(first: %d%s) { nodes { id name key description } pageInfo { hasNextPage endCursor } } }", + remoteScopesPageSize, after, + ) + + res, err := apiClient.Post("", nil, map[string]interface{}{"query": query}, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to query Linear teams") + } + var response linearTeamsGraphqlResponse + if err := api.UnmarshalResponse(res, &response); err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Linear teams response") + } + if len(response.Errors) > 0 { + return nil, nil, errors.Default.New("linear graphql teams query failed: " + response.Errors[0].Message) + } + + return mapLinearTeamsToScopeEntries(response), nextPageFrom(response), nil +} + +// mapLinearTeamsToScopeEntries converts a teams response into scope-list +// entries. Each team is a selectable (leaf) scope. +func mapLinearTeamsToScopeEntries(response linearTeamsGraphqlResponse) []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam] { + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], 0, len(response.Data.Teams.Nodes)) + for _, team := range response.Data.Teams.Nodes { + team := team + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: team.Id, + Name: team.Name, + FullName: team.Name, + Data: &models.LinearTeam{ + TeamId: team.Id, + Name: team.Name, + Key: team.Key, + Description: team.Description, + }, + }) + } + return children +} + +// nextPageFrom returns the cursor for the following page, or nil when the +// teams connection has been fully traversed. +func nextPageFrom(response linearTeamsGraphqlResponse) *LinearRemotePagination { + pageInfo := response.Data.Teams.PageInfo + if pageInfo.HasNextPage && pageInfo.EndCursor != "" { + return &LinearRemotePagination{Cursor: pageInfo.EndCursor} + } + return nil +} + +// RemoteScopes lists the Linear teams available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the Linear API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/linear/api/remote_api_test.go b/backend/plugins/linear/api/remote_api_test.go new file mode 100644 index 00000000000..801e0d5029f --- /dev/null +++ b/backend/plugins/linear/api/remote_api_test.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestMapLinearTeamsToScopeEntries(t *testing.T) { + var response linearTeamsGraphqlResponse + body := `{"data":{"teams":{"nodes":[` + + `{"id":"team-uuid-1","name":"Engineering","key":"ENG","description":"core eng"},` + + `{"id":"team-uuid-2","name":"Design","key":"DSG","description":""}` + + `],"pageInfo":{"hasNextPage":false,"endCursor":""}}}}` + assert.NoError(t, json.Unmarshal([]byte(body), &response)) + + entries := mapLinearTeamsToScopeEntries(response) + assert.Len(t, entries, 2) + + assert.Equal(t, api.RAS_ENTRY_TYPE_SCOPE, entries[0].Type) + assert.Nil(t, entries[0].ParentId) + assert.Equal(t, "team-uuid-1", entries[0].Id) + assert.Equal(t, "Engineering", entries[0].Name) + assert.Equal(t, "Engineering", entries[0].FullName) + // the scope payload must carry the team id used as the scope's primary key + assert.NotNil(t, entries[0].Data) + assert.Equal(t, "team-uuid-1", entries[0].Data.TeamId) + assert.Equal(t, "ENG", entries[0].Data.Key) + + assert.Equal(t, "team-uuid-2", entries[1].Id) + assert.Equal(t, "Design", entries[1].Name) +} + +func TestNextPageFrom(t *testing.T) { + var more linearTeamsGraphqlResponse + more.Data.Teams.PageInfo.HasNextPage = true + more.Data.Teams.PageInfo.EndCursor = "cursor-abc" + next := nextPageFrom(more) + assert.NotNil(t, next) + assert.Equal(t, "cursor-abc", next.Cursor) + + // no further pages -> nil, so the helper stops paginating + var last linearTeamsGraphqlResponse + last.Data.Teams.PageInfo.HasNextPage = false + last.Data.Teams.PageInfo.EndCursor = "cursor-xyz" + assert.Nil(t, nextPageFrom(last)) +} diff --git a/backend/plugins/linear/api/scope_api.go b/backend/plugins/linear/api/scope_api.go new file mode 100644 index 00000000000..9871d1d2bfe --- /dev/null +++ b/backend/plugins/linear/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.LinearTeam] +type ScopeDetail api.ScopeDetail[models.LinearTeam, models.LinearScopeConfig] + +// PutScopes create or update linear teams +// @Summary create or update linear teams +// @Description Create or update linear teams +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to linear team +// @Summary patch to linear team +// @Description patch to linear team +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Param scope body models.LinearTeam true "json" +// @Success 200 {object} models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get linear teams +// @Summary get linear teams +// @Description get linear teams +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one linear team +// @Summary get one linear team +// @Description get one linear team +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/linear/api/scope_config_api.go b/backend/plugins/linear/api/scope_config_api.go new file mode 100644 index 00000000000..e70b800fee2 --- /dev/null +++ b/backend/plugins/linear/api/scope_config_api.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for Linear +// @Summary create scope config for Linear +// @Description create scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for Linear +// @Summary update scope config for Linear +// @Description update scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/linear +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig return projects details related by scope config +// @Summary return all related projects +// @Description return all related projects +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ProjectScopeOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/linear/e2e/account_test.go b/backend/plugins/linear/e2e/account_test.go new file mode 100644 index 00000000000..9cb8f3a8e15 --- /dev/null +++ b/backend/plugins/linear/e2e/account_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearAccountDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_accounts.csv", "_raw_linear_accounts") + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearAccount{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(crossdomain.Account{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/board_test.go b/backend/plugins/linear/e2e/board_test.go new file mode 100644 index 00000000000..eb4e0487323 --- /dev/null +++ b/backend/plugins/linear/e2e/board_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearBoardDataFlow verifies that a Linear team scope is converted into a +// domain ticket.Board, keyed identically to the board_id that board_issues +// already reference (boardIdGen over LinearTeam). Without this, the boards table +// is empty and board-scoped dashboards return no data. +func TestLinearBoardDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the team scope lives in _tool_linear_teams (populated via the scope API) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_teams.csv", &models.LinearTeam{}) + + // convert: team scope -> domain board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertTeamsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/comment_test.go b/backend/plugins/linear/e2e/comment_test.go new file mode 100644 index 00000000000..67ee95adad5 --- /dev/null +++ b/backend/plugins/linear/e2e/comment_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCommentDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the comment convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_comments.csv", "_raw_linear_comments") + dataflowTester.FlushTabler(&models.LinearComment{}) + dataflowTester.Subtask(tasks.ExtractCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueComment{}) + dataflowTester.Subtask(tasks.ConvertCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/cycle_test.go b/backend/plugins/linear/e2e/cycle_test.go new file mode 100644 index 00000000000..207e9c8a3f9 --- /dev/null +++ b/backend/plugins/linear/e2e/cycle_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCycleDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // issues drive sprint_issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_cycles.csv", "_raw_linear_cycles") + dataflowTester.FlushTabler(&models.LinearCycle{}) + dataflowTester.Subtask(tasks.ExtractCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearCycle{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_cycles.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: cycles -> sprints + board_sprints + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Sprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardSprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: issues -> sprint_issues + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_leadtime_test.go b/backend/plugins/linear/e2e/issue_history_leadtime_test.go new file mode 100644 index 00000000000..7d4a4c9798a --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_leadtime_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueHistoryLeadTime proves lead time is derived from the recorded +// state transitions. issue-1 was started on 2026-05-02 and completed on +// 2026-05-03 (1440 min of active cycle time), even though its createdAt -> +// resolutionDate span is 2880 min. ConvertIssues sets the coarse 2880 fallback; +// ConvertIssueHistory must then refine it to 1440 from the history. Issues +// without the required transitions keep the fallback (issue-4 = 1440). +func TestLinearIssueHistoryLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // flush accounts so assignee/creator names stay empty in this lead-time- + // focused test regardless of other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + + // seed issues + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // seed history + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + + // ConvertIssues sets the createdAt -> resolutionDate fallback ... + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + + // ... then ConvertIssueHistory refines lead time from the transitions. + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_history_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_test.go b/backend/plugins/linear/e2e/issue_history_test.go new file mode 100644 index 00000000000..31f559694e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueHistoryDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the history convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssueHistory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_history.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_changelogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_incident_test.go b/backend/plugins/linear/e2e/issue_incident_test.go new file mode 100644 index 00000000000..fb214fa82e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_incident_test.go @@ -0,0 +1,67 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueIncidentMapping verifies label-based issue-type mapping. With a +// scope config whose IssueTypeIncident pattern matches the "Bug" label, ENG-1 +// (labeled "Bug") is converted to an INCIDENT, while issues without a matching +// label stay REQUIREMENT. This is what feeds DORA's change-failure-rate metric. +func TestLinearIssueIncidentMapping(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + IssueTypeIncident: "^Bug$", + }, + } + + // extract issues + inline labels from raw + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // accounts needed so the convertor can resolve assignee/creator names + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // convert -> domain issues; ENG-1 (Bug label) must be INCIDENT, others REQUIREMENT + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_incident.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go new file mode 100644 index 00000000000..783820f696d --- /dev/null +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueNegativeLeadTime guards the lead-time fallback against +// resolution timestamps that precede the creation timestamp (clock skew or +// migrated/imported issues). A negative duration must NOT be cast to uint -- +// doing so wraps to a huge bogus value. The expected behaviour is that no lead +// time is derived (lead_time_minutes stays empty). +func TestLinearIssueNegativeLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // extraction: raw -> tool layer + // Flush accounts so this lead-time-focused test is independent of any + // account rows left behind by other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_negative_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_longtitle_test.go b/backend/plugins/linear/e2e/issue_longtitle_test.go new file mode 100644 index 00000000000..f85062cdb8b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_longtitle_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" + "github.com/stretchr/testify/assert" +) + +// TestLinearIssueLongTitle guards against truncation/insert failure for long +// issue titles and URLs. Linear titles can exceed 255 chars (and the issue URL +// embeds a title slug), which overflowed the old varchar(255) columns and +// failed extraction with "Data too long for column 'title'". The columns are +// now untyped (longtext), matching the domain issues.title and jira's tool +// summary. +func TestLinearIssueLongTitle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ConnectionId: 1, TeamId: "team-1"}, + } + + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_long_title.csv", "_raw_linear_issues") + // must not error with "Data too long for column 'title'" + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + var issue models.LinearIssue + err := dataflowTester.Dal.First(&issue, dal.Where("connection_id = ? AND id = ?", 1, "issue-longtitle")) + assert.NoError(t, err) + assert.Len(t, issue.Title, 300, "full 300-char title must be stored untruncated") +} diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go new file mode 100644 index 00000000000..5f98aad8a61 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer (issues + inline labels) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(models.LinearIssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // accounts must be present so the convertor can resolve assignee/creator + // display names and emit issue_assignees rows. + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // verify conversion: tool layer -> domain layer (issues + board_issues + issue_assignees) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/label_test.go b/backend/plugins/linear/e2e/label_test.go new file mode 100644 index 00000000000..4861f6197da --- /dev/null +++ b/backend/plugins/linear/e2e/label_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearLabelDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // labels are produced inline by the issue extractor + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueLabel{}) + dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv new file mode 100644 index 00000000000..785c47d21b2 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-1"",""Name"":""alice"",""DisplayName"":""Alice Anderson"",""Email"":""alice@example.com"",""AvatarUrl"":""https://linear.app/avatars/alice.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-2"",""Name"":""bob"",""DisplayName"":""Bob Brown"",""Email"":""bob@example.com"",""AvatarUrl"":""https://linear.app/avatars/bob.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-3"",""Name"":""carol"",""DisplayName"":""Carol Clark"",""Email"":""carol@example.com"",""AvatarUrl"":""https://linear.app/avatars/carol.png"",""Active"":false}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv new file mode 100644 index 00000000000..6d6fc21e6a8 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv new file mode 100644 index 00000000000..8eb59ddb145 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-1"",""Number"":1,""Name"":"""",""StartsAt"":""2026-04-20T00:00:00Z"",""EndsAt"":""2026-05-04T00:00:00Z"",""CompletedAt"":""2026-05-04T00:00:00Z""}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-2"",""Number"":2,""Name"":""Sprint 2"",""StartsAt"":""2026-05-04T00:00:00Z"",""EndsAt"":""2026-05-18T00:00:00Z"",""CompletedAt"":null}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv new file mode 100644 index 00000000000..d1cef3c37ab --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv new file mode 100644 index 00000000000..f49f41c73b6 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-1"",""Identifier"":""ENG-1"",""Number"":1,""Title"":""Fix login bug"",""Description"":""Users cannot log in"",""Url"":""https://linear.app/eng/issue/ENG-1"",""Priority"":1,""Estimate"":3,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-03T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":""2026-05-03T00:00:00Z"",""CanceledAt"":null,""State"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""},""Assignee"":{""Id"":""user-1""},""Creator"":{""Id"":""user-2""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l1"",""Name"":""Bug""},{""Id"":""l2"",""Name"":""P1""}]}}",https://api.linear.app/graphql,null,2026-05-03 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-2"",""Identifier"":""ENG-2"",""Number"":2,""Title"":""Add dark mode"",""Description"":""Theme support"",""Url"":""https://linear.app/eng/issue/ENG-2"",""Priority"":2,""Estimate"":5,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l3"",""Name"":""Feature""}]}}",https://api.linear.app/graphql,null,2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-3"",""Identifier"":""ENG-3"",""Number"":3,""Title"":""Investigate flakiness"",""Description"":"""",""Url"":""https://linear.app/eng/issue/ENG-3"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-4"",""Identifier"":""ENG-4"",""Number"":4,""Title"":""Deprecated feature"",""Description"":""No longer needed"",""Url"":""https://linear.app/eng/issue/ENG-4"",""Priority"":3,""Estimate"":2,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T12:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-02T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":{""Id"":""user-3""},""Creator"":{""Id"":""user-1""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-02 12:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-5"",""Identifier"":""ENG-5"",""Number"":5,""Title"":""Write docs"",""Description"":""User guide"",""Url"":""https://linear.app/eng/issue/ENG-5"",""Priority"":4,""Estimate"":1,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T06:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""Assignee"":null,""Creator"":{""Id"":""user-2""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 06:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv new file mode 100644 index 00000000000..4842c33a61c --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-longtitle"",""Identifier"":""ENG-LONG"",""Number"":900,""Title"":""TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"",""Description"":""long title row"",""Url"":""https://linear.app/eng/issue/ENG-LONG/tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":null,""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv new file mode 100644 index 00000000000..f63750db475 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-neg"",""Identifier"":""ENG-NEG"",""Number"":99,""Title"":""Imported issue with skewed timestamps"",""Description"":""canceledAt precedes createdAt"",""Url"":""https://linear.app/eng/issue/ENG-NEG"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-10T00:00:00Z"",""UpdatedAt"":""2026-05-10T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-09T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-10 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv new file mode 100644 index 00000000000..72c0ddc877b --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog"",""Color"":""#bec2c8"",""Position"":0}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted"",""Color"":""#e2e2e2"",""Position"":1}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started"",""Color"":""#f2c94c"",""Position"":2}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed"",""Color"":""#5e6ad2"",""Position"":3}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled"",""Color"":""#95a2b3"",""Position"":4}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv new file mode 100644 index 00000000000..db07f6b4cf7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv @@ -0,0 +1,4 @@ +connection_id,id,name,display_name,email,avatar_url,active +1,user-1,alice,Alice Anderson,alice@example.com,https://linear.app/avatars/alice.png,1 +1,user-2,bob,Bob Brown,bob@example.com,https://linear.app/avatars/bob.png,1 +1,user-3,carol,Carol Clark,carol@example.com,https://linear.app/avatars/carol.png,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv new file mode 100644 index 00000000000..aec4878ecb1 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,body,author_id +1,comment-1,issue-1,Looking into this,user-2 +1,comment-2,issue-1,Fixed in PR 42,user-1 +1,comment-3,issue-2,Any update?,user-1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv new file mode 100644 index 00000000000..9e95c44d986 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv @@ -0,0 +1,3 @@ +connection_id,id,team_id,number,name,starts_at,ends_at,completed_at +1,cycle-1,team-1,1,,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00 +1,cycle-2,team-1,2,Sprint 2,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv new file mode 100644 index 00000000000..bc8fe9ec7ba --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,actor_id,from_state_id,from_state_name,from_state_type,to_state_id,to_state_name,to_state_type +1,hist-1,issue-1,user-2,state-backlog,Backlog,backlog,state-todo,Todo,unstarted +1,hist-2,issue-1,user-1,state-todo,Todo,unstarted,state-inprogress,In Progress,started +1,hist-3,issue-1,user-1,state-inprogress,In Progress,started,state-done,Done,completed diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv new file mode 100644 index 00000000000..20da8a076b0 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv @@ -0,0 +1,4 @@ +connection_id,issue_id,label_name +1,issue-1,Bug +1,issue-1,P1 +1,issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv new file mode 100644 index 00000000000..7c35d53d39e --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv new file mode 100644 index 00000000000..1541f9c95e4 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,team_id,name,key,description +1,0,team-1,Engineering,ENG,Core engineering team diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv new file mode 100644 index 00000000000..bd1d3443ae9 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,name,type,color,position +1,state-backlog,team-1,Backlog,backlog,#bec2c8,0 +1,state-canceled,team-1,Canceled,canceled,#95a2b3,4 +1,state-done,team-1,Done,completed,#5e6ad2,3 +1,state-inprogress,team-1,In Progress,started,#f2c94c,2 +1,state-todo,team-1,Todo,unstarted,#e2e2e2,1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv new file mode 100644 index 00000000000..12c2066d191 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,4 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status +linear:LinearAccount:1:user-1,alice@example.com,Alice Anderson,alice,https://linear.app/avatars/alice.png,,,1 +linear:LinearAccount:1:user-2,bob@example.com,Bob Brown,bob,https://linear.app/avatars/bob.png,,,1 +linear:LinearAccount:1:user-3,carol@example.com,Carol Clark,carol,https://linear.app/avatars/carol.png,,,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9a7d32f9477 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-1 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-2 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-3 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-4 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-5 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..ff0cfe33dc5 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,3 @@ +board_id,sprint_id +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-1 +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-2 diff --git a/backend/plugins/linear/e2e/snapshot_tables/boards.csv b/backend/plugins/linear/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4548d25c385 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +linear:LinearTeam:1:team-1,Engineering,Core engineering team,,,linear diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..38f569f7094 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,3 @@ +issue_id,assignee_id,assignee_name +linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,Alice Anderson +linear:LinearIssue:1:issue-4,linear:LinearAccount:1:user-3,Carol Clark diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..9d53280010f --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,4 @@ +id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date +linear:LinearIssueHistory:1:hist-1,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-2,,state,status,Backlog,Todo,TODO,TODO,2026-05-01T08:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-2,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,Todo,In Progress,TODO,IN_PROGRESS,2026-05-02T00:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-3,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,In Progress,Done,IN_PROGRESS,DONE,2026-05-03T00:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv new file mode 100644 index 00000000000..925e84e5648 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv @@ -0,0 +1,4 @@ +id,issue_id,body,account_id,created_date,updated_date +linear:LinearComment:1:comment-1,linear:LinearIssue:1:issue-1,Looking into this,linear:LinearAccount:1:user-2,2026-05-02T10:00:00.000+00:00,2026-05-02T10:00:00.000+00:00 +linear:LinearComment:1:comment-2,linear:LinearIssue:1:issue-1,Fixed in PR 42,linear:LinearAccount:1:user-1,2026-05-03T09:00:00.000+00:00,2026-05-03T09:30:00.000+00:00 +linear:LinearComment:1:comment-3,linear:LinearIssue:1:issue-2,Any update?,linear:LinearAccount:1:user-1,2026-05-02T11:00:00.000+00:00,2026-05-02T11:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv new file mode 100644 index 00000000000..e7c911e0005 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv @@ -0,0 +1,4 @@ +issue_id,label_name +linear:LinearIssue:1:issue-1,Bug +linear:LinearIssue:1:issue-1,P1 +linear:LinearIssue:1:issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..f55c39e0eb8 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv new file mode 100644 index 00000000000..d11625e3c3d --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv new file mode 100644 index 00000000000..1adc9cab7da --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,INCIDENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv new file mode 100644 index 00000000000..4375d47901b --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-neg,https://linear.app/eng/issue/ENG-NEG,,ENG-NEG,Imported issue with skewed timestamps,canceledAt precedes createdAt,,REQUIREMENT,,DONE,Canceled,,2026-05-09T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..b13b4f85210 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,3 @@ +sprint_id,issue_id +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-1 +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-4 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv new file mode 100644 index 00000000000..1074725b86c --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv @@ -0,0 +1 @@ +sprint_id,issue_id diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..a09ab2ffe70 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,3 @@ +id,name,url,status,started_date,ended_date,completed_date,original_board_id +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 diff --git a/backend/plugins/linear/e2e/sprint_issue_test.go b/backend/plugins/linear/e2e/sprint_issue_test.go new file mode 100644 index 00000000000..81e4508e43f --- /dev/null +++ b/backend/plugins/linear/e2e/sprint_issue_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearSprintIssueStaleCycle guards against stale sprint_issues rows when +// an issue is moved out of a cycle. Sprint membership is derived from the +// issue's cycle_id; once that empties on re-collection, the issue must no +// longer appear in sprint_issues from a previous run. +func TestLinearSprintIssueStaleCycle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // seed issues: issue-1 and issue-4 both belong to cycle-1 + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // first conversion: both issues land in the sprint + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + + // every issue is moved out of its cycle; on re-collection cycle_id empties. + // The second conversion then produces zero sprint issues, which is the case + // the batch divider's lazy delete fails to cover. + if err := dataflowTester.Dal.Exec( + "UPDATE _tool_linear_issues SET cycle_id = '' WHERE connection_id = ? AND team_id = ?", 1, "team-1", + ); err != nil { + t.Fatal(err) + } + + // second conversion (no flush) must drop ALL stale sprint_issues rows + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues_after_leaving_cycle.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/workflow_state_test.go b/backend/plugins/linear/e2e/workflow_state_test.go new file mode 100644 index 00000000000..5af5c4a66b9 --- /dev/null +++ b/backend/plugins/linear/e2e/workflow_state_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearWorkflowStateDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_workflow_states.csv", "_raw_linear_workflow_states") + dataflowTester.FlushTabler(&models.LinearWorkflowState{}) + dataflowTester.Subtask(tasks.ExtractWorkflowStatesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearWorkflowState{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_workflow_states.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go new file mode 100644 index 00000000000..e1bc9849131 --- /dev/null +++ b/backend/plugins/linear/impl/impl.go @@ -0,0 +1,233 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Linear)(nil) + +type Linear struct{} + +func (p Linear) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Linear) Description() string { + return "To collect and enrich data from Linear" +} + +func (p Linear) Name() string { + return "linear" +} + +func (p Linear) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linear" +} + +func (p Linear) Connection() dal.Tabler { + return &models.LinearConnection{} +} + +func (p Linear) Scope() plugin.ToolLayerScope { + return &models.LinearTeam{} +} + +func (p Linear) ScopeConfig() dal.Tabler { + return &models.LinearScopeConfig{} +} + +func (p Linear) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linear) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.LinearConnection{}, + &models.LinearTeam{}, + &models.LinearScopeConfig{}, + &models.LinearAccount{}, + &models.LinearIssue{}, + &models.LinearComment{}, + &models.LinearIssueLabel{}, + &models.LinearWorkflowState{}, + &models.LinearCycle{}, + &models.LinearIssueHistory{}, + } +} + +func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectAccountsMeta, + tasks.ExtractAccountsMeta, + tasks.CollectWorkflowStatesMeta, + tasks.ExtractWorkflowStatesMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, + tasks.CollectCommentsMeta, + tasks.ExtractCommentsMeta, + tasks.CollectCyclesMeta, + tasks.ExtractCyclesMeta, + tasks.CollectIssueHistoryMeta, + tasks.ExtractIssueHistoryMeta, + tasks.ConvertTeamsMeta, + tasks.ConvertAccountsMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, + tasks.ConvertCommentsMeta, + tasks.ConvertCyclesMeta, + tasks.ConvertSprintIssuesMeta, + tasks.ConvertIssueHistoryMeta, + } +} + +func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.LinearOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode Linear options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("linear connectionId is invalid") + } + if op.TeamId == "" { + return nil, errors.BadInput.New("linear teamId is required") + } + + connection := &models.LinearConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Linear plugin") + } + + graphqlClient, err := tasks.NewLinearGraphqlClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create Linear GraphQL client") + } + + // Resolve the scope config (label-based issue-type mapping). Default to an + // empty config when none is set so subtasks can rely on it being non-nil. + scopeConfig := &models.LinearScopeConfig{} + if op.ScopeConfigId != 0 { + if err := taskCtx.GetDal().First(scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); err != nil { + return nil, errors.Default.Wrap(err, "error getting scope config for Linear plugin") + } + } + + taskData := &tasks.LinearTaskData{ + Options: &op, + GraphqlClient: graphqlClient, + ScopeConfig: scopeConfig, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Linear) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Linear) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.LinearTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.GraphqlClient != nil { + data.GraphqlClient.Release() + } + return nil +} diff --git a/backend/plugins/linear/impl/impl_test.go b/backend/plugins/linear/impl/impl_test.go new file mode 100644 index 00000000000..d11b3ec5ae1 --- /dev/null +++ b/backend/plugins/linear/impl/impl_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestApiResourcesRegistersRemoteScopes guards the remote-scopes endpoints used +// by the config UI to enumerate Linear teams. +func TestApiResourcesRegistersRemoteScopes(t *testing.T) { + resources := Linear{}.ApiResources() + + remoteScopes, ok := resources["connections/:connectionId/remote-scopes"] + assert.True(t, ok, "remote-scopes route must be registered") + assert.NotNil(t, remoteScopes["GET"], "remote-scopes must handle GET") + + proxy, ok := resources["connections/:connectionId/proxy/rest/*path"] + assert.True(t, ok, "proxy route must be registered") + assert.NotNil(t, proxy["GET"], "proxy must handle GET") +} diff --git a/backend/plugins/linear/linear.go b/backend/plugins/linear/linear.go new file mode 100644 index 00000000000..2cb64a49470 --- /dev/null +++ b/backend/plugins/linear/linear.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Linear //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "linear"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "linear connection id") + teamId := cmd.Flags().StringP("team", "t", "", "linear team id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("team") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "teamId": *teamId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linear/models/account.go b/backend/plugins/linear/models/account.go new file mode 100644 index 00000000000..7d1c9376f98 --- /dev/null +++ b/backend/plugins/linear/models/account.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearAccount is a Linear user (tool layer), converted to crossdomain.Account. +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Name string `gorm:"type:varchar(255)" json:"name"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Email string `gorm:"type:varchar(255)" json:"email"` + AvatarUrl string `gorm:"type:varchar(255)" json:"avatarUrl"` + Active bool `json:"active"` + common.NoPKModel +} + +func (LinearAccount) TableName() string { + return "_tool_linear_accounts" +} diff --git a/backend/plugins/linear/models/connection.go b/backend/plugins/linear/models/connection.go new file mode 100644 index 00000000000..6f63e431a11 --- /dev/null +++ b/backend/plugins/linear/models/connection.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// LinearConn holds the essential information to connect to the Linear API. +// Linear authenticates with a personal API key passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. +type LinearConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the Linear API. +func (lc *LinearConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", lc.Token) + return nil +} + +func (lc *LinearConn) Sanitize() LinearConn { + lc.Token = utils.SanitizeString(lc.Token) + return *lc +} + +// LinearConnection holds LinearConn plus ID/Name for database storage. +type LinearConnection struct { + helper.BaseConnection `mapstructure:",squash"` + LinearConn `mapstructure:",squash"` +} + +func (connection LinearConnection) Sanitize() LinearConnection { + connection.LinearConn = connection.LinearConn.Sanitize() + return connection +} + +func (connection *LinearConnection) MergeFromRequest(target *LinearConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (LinearConnection) TableName() string { + return "_tool_linear_connections" +} diff --git a/backend/plugins/linear/models/cycle.go b/backend/plugins/linear/models/cycle.go new file mode 100644 index 00000000000..4f61370dbb2 --- /dev/null +++ b/backend/plugins/linear/models/cycle.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearCycle is a Linear cycle (sprint-equivalent), converted to ticket.Sprint. +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Number int `json:"number"` + Name string `gorm:"type:varchar(255)" json:"name"` + StartsAt *time.Time `json:"startsAt"` + EndsAt *time.Time `json:"endsAt"` + CompletedAt *time.Time `json:"completedAt"` + common.NoPKModel +} + +func (LinearCycle) TableName() string { + return "_tool_linear_cycles" +} diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go new file mode 100644 index 00000000000..991585de956 --- /dev/null +++ b/backend/plugins/linear/models/issue.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssue is the tool-layer representation of a Linear issue. +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `json:"title"` + Description string `json:"description"` + Url string `json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` + common.NoPKModel +} + +func (LinearIssue) TableName() string { + return "_tool_linear_issues" +} diff --git a/backend/plugins/linear/models/issue_comment.go b/backend/plugins/linear/models/issue_comment.go new file mode 100644 index 00000000000..8ba18505038 --- /dev/null +++ b/backend/plugins/linear/models/issue_comment.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearComment is the tool-layer representation of a comment on a Linear issue. +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + Body string `json:"body"` + AuthorId string `gorm:"type:varchar(255)" json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + common.NoPKModel +} + +func (LinearComment) TableName() string { + return "_tool_linear_comments" +} diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go new file mode 100644 index 00000000000..027f6cb4a07 --- /dev/null +++ b/backend/plugins/linear/models/issue_history.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueHistory is a single entry in a Linear issue's history, used to +// build domain-layer changelogs and derive lead/cycle time. +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + ActorId string `gorm:"type:varchar(255)" json:"actorId"` + FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` + FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + FromStateType string `gorm:"type:varchar(100)" json:"fromStateType"` + ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` + ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + ToStateType string `gorm:"type:varchar(100)" json:"toStateType"` + CreatedAt time.Time `gorm:"index" json:"createdAt"` + common.NoPKModel +} + +func (LinearIssueHistory) TableName() string { + return "_tool_linear_issue_history" +} diff --git a/backend/plugins/linear/models/issue_label.go b/backend/plugins/linear/models/issue_label.go new file mode 100644 index 00000000000..76a4517cb7b --- /dev/null +++ b/backend/plugins/linear/models/issue_label.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueLabel joins a Linear issue to one of its labels. Labels are +// collected inline with issues, so there is no separate label collector. +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)" json:"issueId"` + LabelName string `gorm:"primaryKey;type:varchar(255)" json:"labelName"` + common.NoPKModel +} + +func (LinearIssueLabel) TableName() string { + return "_tool_linear_issue_labels" +} diff --git a/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go new file mode 100644 index 00000000000..9bb577343be --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.LinearConnection{}, + &archived.LinearTeam{}, + &archived.LinearScopeConfig{}, + &archived.LinearAccount{}, + &archived.LinearIssue{}, + &archived.LinearComment{}, + &archived.LinearIssueLabel{}, + &archived.LinearWorkflowState{}, + &archived.LinearCycle{}, + &archived.LinearIssueHistory{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260601000001 +} + +func (*addInitTables) Name() string { + return "linear init schemas" +} diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..a90c3217153 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -0,0 +1,169 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/linear/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type LinearConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (LinearConnection) TableName() string { return "_tool_linear_connections" } + +type LinearTeam struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + TeamId string `json:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + Key string `json:"key" gorm:"type:varchar(255)"` + Description string `json:"description"` +} + +func (LinearTeam) TableName() string { return "_tool_linear_teams" } + +type LinearScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + IssueTypeRequirement string `json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { return "_tool_linear_scope_configs" } + +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + DisplayName string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + Active bool + archived.NoPKModel +} + +func (LinearAccount) TableName() string { return "_tool_linear_accounts" } + +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string + Description string + Url string + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + archived.NoPKModel +} + +func (LinearIssue) TableName() string { return "_tool_linear_issues" } + +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + Body string + AuthorId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearComment) TableName() string { return "_tool_linear_comments" } + +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + LabelName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (LinearIssueLabel) TableName() string { return "_tool_linear_issue_labels" } + +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Type string `gorm:"type:varchar(100)"` + Color string `gorm:"type:varchar(50)"` + Position float64 + archived.NoPKModel +} + +func (LinearWorkflowState) TableName() string { return "_tool_linear_workflow_states" } + +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Number int + Name string `gorm:"type:varchar(255)"` + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time + archived.NoPKModel +} + +func (LinearCycle) TableName() string { return "_tool_linear_cycles" } + +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + ActorId string `gorm:"type:varchar(255)"` + FromStateId string `gorm:"type:varchar(255)"` + FromStateName string `gorm:"type:varchar(255)"` + FromStateType string `gorm:"type:varchar(100)"` + ToStateId string `gorm:"type:varchar(255)"` + ToStateName string `gorm:"type:varchar(255)"` + ToStateType string `gorm:"type:varchar(100)"` + CreatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearIssueHistory) TableName() string { return "_tool_linear_issue_history" } diff --git a/backend/plugins/linear/models/migrationscripts/register.go b/backend/plugins/linear/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/linear/models/scope_config.go b/backend/plugins/linear/models/scope_config.go new file mode 100644 index 00000000000..309ce749eae --- /dev/null +++ b/backend/plugins/linear/models/scope_config.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearScopeConfig keeps status mapping deterministic (Linear's +// WorkflowState.type maps to TODO/IN_PROGRESS/DONE without user input) but +// allows label-based issue-type mapping. Linear has no native issue "type", so +// each pattern below is a regular expression matched against an issue's label +// names to derive the domain ticket.Issue.Type. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue matching none defaults to REQUIREMENT. +type LinearScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { + return "_tool_linear_scope_configs" +} + +func (sc *LinearScopeConfig) SetConnectionId(c *LinearScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/linear/models/team.go b/backend/plugins/linear/models/team.go new file mode 100644 index 00000000000..d6226208266 --- /dev/null +++ b/backend/plugins/linear/models/team.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*LinearTeam)(nil) + +// LinearTeam is the data-source scope for the Linear plugin. A Linear Team +// owns issues, cycles, workflow states and labels, mapping cleanly to a +// DevLake domain-layer ticket.Board. +type LinearTeam struct { + common.Scope `mapstructure:",squash"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description"` +} + +func (t LinearTeam) ScopeId() string { + return t.TeamId +} + +func (t LinearTeam) ScopeName() string { + return t.Name +} + +func (t LinearTeam) ScopeFullName() string { + return t.Name +} + +func (t LinearTeam) ScopeParams() interface{} { + return &LinearApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (LinearTeam) TableName() string { + return "_tool_linear_teams" +} + +// LinearApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_linear_* table. +type LinearApiParams struct { + ConnectionId uint64 + TeamId string +} diff --git a/backend/plugins/linear/models/workflow_state.go b/backend/plugins/linear/models/workflow_state.go new file mode 100644 index 00000000000..9273f183c91 --- /dev/null +++ b/backend/plugins/linear/models/workflow_state.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearWorkflowState is a Linear team's workflow state. Its Type +// (backlog|unstarted|started|completed|canceled) drives issue status mapping. +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Type string `gorm:"type:varchar(100)" json:"type"` + Color string `gorm:"type:varchar(50)" json:"color"` + Position float64 `json:"position"` + common.NoPKModel +} + +func (LinearWorkflowState) TableName() string { + return "_tool_linear_workflow_states" +} diff --git a/backend/plugins/linear/tasks/account_collector.go b/backend/plugins/linear/tasks/account_collector.go new file mode 100644 index 00000000000..88fa181f002 --- /dev/null +++ b/backend/plugins/linear/tasks/account_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ACCOUNTS_TABLE = "linear_accounts" + +// GraphqlQueryAccountWrapper is the paginated `users` query envelope. +type GraphqlQueryAccountWrapper struct { + Users struct { + Nodes []GraphqlQueryAccount + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"users(first: $pageSize, after: $skipCursor)"` +} + +type GraphqlQueryAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string + Active bool +} + +var CollectAccountsMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectAccounts, + EnabledByDefault: true, + Description: "Collect workspace users from the Linear GraphQL API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectAccounts + +func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryAccountWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryAccountWrapper) + return query.Users.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryAccountWrapper) + for _, account := range query.Users.Nodes { + messages = append(messages, errors.Must1(json.Marshal(account))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/account_convertor.go b/backend/plugins/linear/tasks/account_convertor.go new file mode 100644 index 00000000000..d3c8efd3686 --- /dev/null +++ b/backend/plugins/linear/tasks/account_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_accounts into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.LinearAccount{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertAccounts + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearAccount{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearAccount{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + account := inputRow.(*models.LinearAccount) + status := 1 + if !account.Active { + status = 0 + } + fullName := account.Name + if account.DisplayName != "" { + fullName = account.DisplayName + } + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, account.Id), + }, + UserName: account.Name, + FullName: fullName, + Email: account.Email, + AvatarUrl: account.AvatarUrl, + Status: status, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/account_extractor.go b/backend/plugins/linear/tasks/account_extractor.go new file mode 100644 index 00000000000..19cc86a04cc --- /dev/null +++ b/backend/plugins/linear/tasks/account_extractor.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractAccountsMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractAccounts, + EnabledByDefault: true, + Description: "Extract raw user data into tool layer table _tool_linear_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractAccounts + +func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiAccount := &GraphqlQueryAccount{} + if err := errors.Convert(json.Unmarshal(row.Data, apiAccount)); err != nil { + return nil, err + } + if apiAccount.Id == "" { + return nil, nil + } + account := &models.LinearAccount{ + ConnectionId: data.Options.ConnectionId, + Id: apiAccount.Id, + Name: apiAccount.Name, + DisplayName: apiAccount.DisplayName, + Email: apiAccount.Email, + AvatarUrl: apiAccount.AvatarUrl, + Active: apiAccount.Active, + } + return []interface{}{account}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/api_client.go b/backend/plugins/linear/tasks/api_client.go new file mode 100644 index 00000000000..0ca6b8da25a --- /dev/null +++ b/backend/plugins/linear/tasks/api_client.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + gocontext "context" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +// linearTransport injects the Linear personal API key into every request. +// Linear expects the key verbatim in the Authorization header (no Bearer prefix). +type linearTransport struct { + token string + base http.RoundTripper +} + +func (t *linearTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", t.token) + return t.base.RoundTrip(req) +} + +// graphqlQueryViewer is a tiny probe used to validate connectivity / liveness. +type graphqlQueryViewer struct { + Viewer struct { + Id graphql.String + } +} + +// defaultRateLimitPerHour is Linear's documented per-API-key request budget. +// Used when the connection does not override RateLimitPerHour. +const defaultRateLimitPerHour = 1500 + +// NewLinearGraphqlClient builds a rate-limited async GraphQL client for the +// Linear API from the given connection. +func NewLinearGraphqlClient(taskCtx plugin.TaskContext, connection *models.LinearConnection) (*helper.GraphqlAsyncClient, errors.Error) { + httpClient, err := newLinearHttpClient(connection) + if err != nil { + return nil, err + } + + endpoint := connection.Endpoint + if endpoint == "" { + endpoint = "https://api.linear.app/graphql" + } + client := graphql.NewClient(endpoint, httpClient) + + rateLimitPerHour := connection.RateLimitPerHour + if rateLimitPerHour <= 0 { + rateLimitPerHour = defaultRateLimitPerHour + } + + return helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + func(ctx gocontext.Context, c *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, e errors.Error) { + // Linear does not expose rate-limit info in the GraphQL body (it uses + // HTTP response headers), so we probe liveness and pace against the + // configured hourly budget. The async client self-throttles from here. + var q graphqlQueryViewer + dataErrors, queryErr := errors.Convert01(c.Query(ctx, &q, nil)) + if queryErr != nil { + return 0, nil, queryErr + } + if len(dataErrors) > 0 { + return 0, nil, errors.Default.Wrap(dataErrors[0], "linear graphql viewer query failed") + } + reset := time.Now().Add(1 * time.Hour) + logger.Info("linear graphql client initialized, pacing against %d req/hour", rateLimitPerHour) + return rateLimitPerHour, &reset, nil + }) +} + +func newLinearHttpClient(connection *models.LinearConnection) (*http.Client, errors.Error) { + base := http.DefaultTransport + if proxy := connection.Proxy; proxy != "" { + pu, err := url.Parse(proxy) + if err != nil { + return nil, errors.BadInput.Wrap(err, "malformed proxy url") + } + base = &http.Transport{Proxy: http.ProxyURL(pu)} + } + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &linearTransport{token: connection.Token, base: base}, + }, nil +} diff --git a/backend/plugins/linear/tasks/board_convertor.go b/backend/plugins/linear/tasks/board_convertor.go new file mode 100644 index 00000000000..9366efed2ab --- /dev/null +++ b/backend/plugins/linear/tasks/board_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// RAW_TEAMS_TABLE labels the raw-data lineage for the team-scope-derived board. +// Teams are added as scopes (no collector), so this is a logical tag only. +const RAW_TEAMS_TABLE = "linear_teams" + +var ConvertTeamsMeta = plugin.SubTaskMeta{ + Name: "Convert Teams", + EntryPoint: ConvertTeams, + EnabledByDefault: true, + Description: "Convert the Linear team scope (_tool_linear_teams) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearTeam{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTeams + +func ConvertTeams(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the issue/sprint convertors so the + // board joins to the board_issues/sprint_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearTeam{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_TEAMS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearTeam{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + team := inputRow.(*models.LinearTeam) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, team.TeamId)}, + Name: team.Name, + Description: team.Description, + Type: "linear", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go new file mode 100644 index 00000000000..14e801ff9e3 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -0,0 +1,148 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_COMMENTS_TABLE = "linear_comments" + +// SimpleLinearIssue is the iterator element used to drive per-issue collection +// of child resources (comments, history). Its JSON form is stored in the raw +// row's `input` column so extractors can recover the owning issue id. +type SimpleLinearIssue struct { + // Id is populated by the DalCursorIterator (the _tool_linear_issues.id column) + // when driving per-issue child collection. + Id string `json:"Id"` + // IssueId is populated when parsing a raw row's `input` column: the GraphQL + // collector stores the query variables there (which carry `issueId`), not the + // iterator element. OwningIssueId resolves whichever is present. + IssueId string `json:"issueId" gorm:"-"` +} + +// OwningIssueId returns the issue id this child row belongs to, tolerating both +// the iterator element shape ({"Id":...}) and the collector's stored variables +// shape ({"issueId":...}). +func (s SimpleLinearIssue) OwningIssueId() string { + if s.IssueId != "" { + return s.IssueId + } + return s.Id +} + +// GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. +type GraphqlQueryCommentWrapper struct { + Issue struct { + Comments struct { + Nodes []GraphqlQueryComment + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"comments(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryComment struct { + Id string + Body string + CreatedAt time.Time + UpdatedAt time.Time + User *struct{ Id string } +} + +var CollectCommentsMeta = plugin.SubTaskMeta{ + Name: "Collect Comments", + EntryPoint: CollectComments, + EnabledByDefault: true, + Description: "Collect comments for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectComments + +func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's comments cannot have changed, so re-fetching every + // issue each run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCommentWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCommentWrapper) + return query.Issue.Comments.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCommentWrapper) + for _, comment := range query.Issue.Comments.Nodes { + messages = append(messages, errors.Must1(json.Marshal(comment))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_convertor.go b/backend/plugins/linear/tasks/comment_convertor.go new file mode 100644 index 00000000000..8c4729f0e5c --- /dev/null +++ b/backend/plugins/linear/tasks/comment_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCommentsMeta = plugin.SubTaskMeta{ + Name: "Convert Comments", + EntryPoint: ConvertComments, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_comments into domain layer table issue_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearComment{}.TableName(), RAW_COMMENTS_TABLE}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertComments + +func ConvertComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + commentIdGen := didgen.NewDomainIdGenerator(&models.LinearComment{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("c.*"), + dal.From("_tool_linear_comments c"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = c.connection_id AND i.id = c.issue_id)"), + dal.Where("c.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearComment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + comment := inputRow.(*models.LinearComment) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(connectionId, comment.Id)}, + IssueId: issueIdGen.Generate(connectionId, comment.IssueId), + Body: comment.Body, + CreatedDate: comment.CreatedAt, + } + if comment.AuthorId != "" { + domainComment.AccountId = accountIdGen.Generate(connectionId, comment.AuthorId) + } + if !comment.UpdatedAt.IsZero() { + domainComment.UpdatedDate = &comment.UpdatedAt + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go new file mode 100644 index 00000000000..15b28a3b969 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCommentsMeta = plugin.SubTaskMeta{ + Name: "Extract Comments", + EntryPoint: ExtractComments, + EnabledByDefault: true, + Description: "Extract raw comment data into tool layer table _tool_linear_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractComments + +func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiComment := &GraphqlQueryComment{} + if err := errors.Convert(json.Unmarshal(row.Data, apiComment)); err != nil { + return nil, err + } + // The owning issue id is carried in the raw row's input column. + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + comment := &models.LinearComment{ + ConnectionId: data.Options.ConnectionId, + Id: apiComment.Id, + IssueId: issueRef.OwningIssueId(), + Body: apiComment.Body, + CreatedAt: apiComment.CreatedAt, + UpdatedAt: apiComment.UpdatedAt, + } + if apiComment.User != nil { + comment.AuthorId = apiComment.User.Id + } + return []interface{}{comment}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_collector.go b/backend/plugins/linear/tasks/cycle_collector.go new file mode 100644 index 00000000000..88572650fcd --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_CYCLES_TABLE = "linear_cycles" + +// GraphqlQueryCycleWrapper is the team-scoped, paginated `cycles` query. +type GraphqlQueryCycleWrapper struct { + Team struct { + Cycles struct { + Nodes []GraphqlQueryCycle + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"cycles(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryCycle struct { + Id string + Number int + Name string + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time +} + +var CollectCyclesMeta = plugin.SubTaskMeta{ + Name: "Collect Cycles", + EntryPoint: CollectCycles, + EnabledByDefault: true, + Description: "Collect cycles (sprints) for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectCycles + +func CollectCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCycleWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCycleWrapper) + return query.Team.Cycles.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCycleWrapper) + for _, cycle := range query.Team.Cycles.Nodes { + messages = append(messages, errors.Must1(json.Marshal(cycle))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_convertor.go b/backend/plugins/linear/tasks/cycle_convertor.go new file mode 100644 index 00000000000..ae4a0dd5a84 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCyclesMeta = plugin.SubTaskMeta{ + Name: "Convert Cycles", + EntryPoint: ConvertCycles, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_cycles into domain layer tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearCycle{}.TableName(), RAW_CYCLES_TABLE}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertCycles + +func ConvertCycles(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearCycle{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearCycle{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + cycle := inputRow.(*models.LinearCycle) + sprintId := cycleIdGen.Generate(connectionId, cycle.Id) + name := cycle.Name + if name == "" { + name = fmt.Sprintf("Cycle %d", cycle.Number) + } + status := "ACTIVE" + if cycle.CompletedAt != nil { + status = "CLOSED" + } + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: name, + Status: status, + StartedDate: cycle.StartsAt, + EndedDate: cycle.EndsAt, + CompletedDate: cycle.CompletedAt, + OriginalBoardID: boardId, + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_extractor.go b/backend/plugins/linear/tasks/cycle_extractor.go new file mode 100644 index 00000000000..43a4233aa88 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_extractor.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCyclesMeta = plugin.SubTaskMeta{ + Name: "Extract Cycles", + EntryPoint: ExtractCycles, + EnabledByDefault: true, + Description: "Extract raw cycle data into tool layer table _tool_linear_cycles", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractCycles + +func ExtractCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiCycle := &GraphqlQueryCycle{} + if err := errors.Convert(json.Unmarshal(row.Data, apiCycle)); err != nil { + return nil, err + } + cycle := &models.LinearCycle{ + ConnectionId: data.Options.ConnectionId, + Id: apiCycle.Id, + TeamId: data.Options.TeamId, + Number: apiCycle.Number, + Name: apiCycle.Name, + StartsAt: apiCycle.StartsAt, + EndsAt: apiCycle.EndsAt, + CompletedAt: apiCycle.CompletedAt, + } + return []interface{}{cycle}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go new file mode 100644 index 00000000000..9f5a0aedd1d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUES_TABLE = "linear_issues" + +// GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. +// Incremental runs filter server-side on updatedAt ($filter) rather than +// relying on result ordering, so collection no longer depends on an undocumented +// default sort direction. +type GraphqlQueryIssueWrapper struct { + Team struct { + Issues struct { + Nodes []GraphqlQueryIssue `graphql:"nodes"` + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt, filter: $filter)"` + } `graphql:"team(id: $teamId)"` +} + +// IssueFilter mirrors the subset of Linear's GraphQL IssueFilter input used to +// restrict collection to issues updated after a point in time. The Go type +// name is significant: the GraphQL client emits it as the variable's type +// ($filter:IssueFilter!). +type IssueFilter struct { + UpdatedAt *DateComparator `json:"updatedAt,omitempty"` +} + +// DateComparator mirrors Linear's DateComparator input (only the `gt` operator +// is needed here). +type DateComparator struct { + Gt *time.Time `json:"gt,omitempty"` +} + +// buildIssueFilter returns an IssueFilter restricting to issues updated after +// `since`. When `since` is nil (a full sync) it returns the empty filter, which +// Linear treats as "match all". +func buildIssueFilter(since *time.Time) IssueFilter { + if since == nil { + return IssueFilter{} + } + return IssueFilter{UpdatedAt: &DateComparator{Gt: since}} +} + +type GraphqlQueryIssue struct { + Id string + Identifier string + Number int + Title string + Description string + Url string + Priority int + Estimate *float64 + CreatedAt time.Time + UpdatedAt time.Time + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + State *struct { + Id string + Name string + Type string + } + Assignee *struct{ Id string } + Creator *struct{ Id string } + Cycle *struct{ Id string } + Parent *struct{ Id string } + Labels struct { + Nodes []struct { + Id string + Name string + } + } `graphql:"labels(first: 50)"` +} + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "Collect Issues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "Collect issues for a Linear team, supports incremental collection", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssues + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }) + if err != nil { + return err + } + + since := apiCollector.GetSince() + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryIssueWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + "filter": buildIssueFilter(since), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryIssueWrapper) + return query.Team.Issues.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryIssueWrapper) + // The server-side $filter already restricts to issues updated after + // `since`, so every returned issue is in scope -- no client-side + // early-stop (and thus no dependency on sort direction) is needed. + for _, issue := range query.Team.Issues.Nodes { + issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) + issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) + issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) + messages = append(messages, errors.Must1(json.Marshal(issue))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector_test.go b/backend/plugins/linear/tasks/issue_collector_test.go new file mode 100644 index 00000000000..2dbbeecf586 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestBuildIssueFilter pins the server-side incremental filter that replaces +// the previous reliance on result ordering plus a client-side early-stop. A +// full sync must produce an empty filter (match all); an incremental run must +// produce Linear's IssueFilter shape `{ updatedAt: { gt: } }`. +func TestBuildIssueFilter(t *testing.T) { + full, err := json.Marshal(buildIssueFilter(nil)) + assert.NoError(t, err) + assert.Equal(t, "{}", string(full)) + + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental, err := json.Marshal(buildIssueFilter(&since)) + assert.NoError(t, err) + assert.JSONEq(t, `{"updatedAt":{"gt":"2026-05-01T00:00:00Z"}}`, string(incremental)) +} diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go new file mode 100644 index 00000000000..b124e59f369 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -0,0 +1,229 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Issues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssues + +func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + // Preload account display names so issues can carry assignee/creator names + // and emit issue_assignees rows, mirroring how the account convertor derives + // the domain account's full name (displayName, falling back to name). + var accounts []models.LinearAccount + if err := db.All(&accounts, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + accountNames := make(map[string]string, len(accounts)) + for _, account := range accounts { + name := account.Name + if account.DisplayName != "" { + name = account.DisplayName + } + accountNames[account.Id] = name + } + + // Preload issue labels so the convertor can derive domain issue Type from a + // label-based mapping (Linear has no native issue type). + var issueLabels []models.LinearIssueLabel + if err := db.All(&issueLabels, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + labelsByIssue := make(map[string][]string, len(issueLabels)) + for _, l := range issueLabels { + labelsByIssue[l.IssueId] = append(labelsByIssue[l.IssueId], l.LabelName) + } + + // Compile the label-matching patterns from the scope config. An empty + // pattern is treated as "no match" (nil), so issues default to REQUIREMENT. + typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) + if err != nil { + return err + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: typeMatcher.typeOf(labelsByIssue[issue.Id]), + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, + } + if issue.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + domainIssue.CreatorName = accountNames[issue.CreatorId] + } + if issue.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + domainIssue.AssigneeName = accountNames[issue.AssigneeId] + } + if issue.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) + domainIssue.IsSubtask = true + } + // Resolution date: completedAt, falling back to canceledAt. + if issue.CompletedAt != nil { + domainIssue.ResolutionDate = issue.CompletedAt + } else if issue.CanceledAt != nil { + domainIssue.ResolutionDate = issue.CanceledAt + } + // Fallback lead time when no history-derived value is present. + // Guard against a resolution that precedes creation (clock skew or + // migrated/imported issues): a negative duration cast to uint yields + // platform-dependent garbage, so leave lead time unset instead. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil && + domainIssue.ResolutionDate.After(issue.CreatedAt) { + minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// issueTypeMatcher derives the domain ticket.Issue.Type from an issue's label +// names using the scope config's regex patterns. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue whose labels match none (or with no +// patterns configured) defaults to REQUIREMENT. +type issueTypeMatcher struct { + incident *regexp.Regexp + bug *regexp.Regexp + requirement *regexp.Regexp +} + +func newIssueTypeMatcher(sc *models.LinearScopeConfig) (*issueTypeMatcher, errors.Error) { + m := &issueTypeMatcher{} + if sc == nil { + return m, nil + } + for _, p := range []struct { + pattern string + field string + out **regexp.Regexp + }{ + {sc.IssueTypeIncident, "issueTypeIncident", &m.incident}, + {sc.IssueTypeBug, "issueTypeBug", &m.bug}, + {sc.IssueTypeRequirement, "issueTypeRequirement", &m.requirement}, + } { + if p.pattern == "" { + continue + } + re, err := regexp.Compile(p.pattern) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid "+p.field+" pattern") + } + *p.out = re + } + return m, nil +} + +func (m *issueTypeMatcher) typeOf(labels []string) string { + for _, c := range []struct { + pattern *regexp.Regexp + typ string + }{ + {m.incident, ticket.INCIDENT}, + {m.bug, ticket.BUG}, + {m.requirement, ticket.REQUIREMENT}, + } { + if c.pattern == nil { + continue + } + for _, name := range labels { + if c.pattern.MatchString(name) { + return c.typ + } + } + } + return ticket.REQUIREMENT +} diff --git a/backend/plugins/linear/tasks/issue_extractor.go b/backend/plugins/linear/tasks/issue_extractor.go new file mode 100644 index 00000000000..e423076612d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_extractor.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "Extract Issues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "Extract raw issue data into tool layer tables _tool_linear_issues and _tool_linear_issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiIssue := &GraphqlQueryIssue{} + if err := errors.Convert(json.Unmarshal(row.Data, apiIssue)); err != nil { + return nil, err + } + connectionId := data.Options.ConnectionId + issue := &models.LinearIssue{ + ConnectionId: connectionId, + Id: apiIssue.Id, + TeamId: data.Options.TeamId, + Identifier: apiIssue.Identifier, + Number: apiIssue.Number, + Title: apiIssue.Title, + Description: apiIssue.Description, + Url: apiIssue.Url, + Priority: apiIssue.Priority, + PriorityLabel: PriorityLabel(apiIssue.Priority), + Estimate: apiIssue.Estimate, + CreatedAt: apiIssue.CreatedAt, + UpdatedAt: apiIssue.UpdatedAt, + StartedAt: apiIssue.StartedAt, + CompletedAt: apiIssue.CompletedAt, + CanceledAt: apiIssue.CanceledAt, + } + if apiIssue.State != nil { + issue.StateId = apiIssue.State.Id + issue.StateName = apiIssue.State.Name + issue.StateType = apiIssue.State.Type + } + if apiIssue.Assignee != nil { + issue.AssigneeId = apiIssue.Assignee.Id + } + if apiIssue.Creator != nil { + issue.CreatorId = apiIssue.Creator.Id + } + if apiIssue.Cycle != nil { + issue.CycleId = apiIssue.Cycle.Id + } + if apiIssue.Parent != nil { + issue.ParentId = apiIssue.Parent.Id + } + + results := make([]interface{}, 0, len(apiIssue.Labels.Nodes)+1) + results = append(results, issue) + for _, label := range apiIssue.Labels.Nodes { + results = append(results, &models.LinearIssueLabel{ + ConnectionId: connectionId, + IssueId: apiIssue.Id, + LabelName: label.Name, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go new file mode 100644 index 00000000000..11de7c218c7 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUE_HISTORY_TABLE = "linear_issue_history" + +// GraphqlQueryHistoryWrapper is the per-issue, paginated `history` query. +type GraphqlQueryHistoryWrapper struct { + Issue struct { + History struct { + Nodes []GraphqlQueryHistory + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"history(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryHistory struct { + Id string + CreatedAt time.Time + Actor *struct{ Id string } + FromState *struct { + Id string + Name string + Type string + } + ToState *struct { + Id string + Name string + Type string + } +} + +var CollectIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Collect Issue History", + EntryPoint: CollectIssueHistory, + EnabledByDefault: true, + Description: "Collect history events for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssueHistory + +func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's history cannot have changed, so re-fetching it every + // run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryHistoryWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryHistoryWrapper) + return query.Issue.History.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryHistoryWrapper) + for _, event := range query.Issue.History.Nodes { + // Only state transitions are relevant to the status changelog. + if event.FromState == nil && event.ToState == nil { + continue + } + messages = append(messages, errors.Must1(json.Marshal(event))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go new file mode 100644 index 00000000000..f1fa99d0519 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Convert Issue History", + EntryPoint: ConvertIssueHistory, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_history into domain layer table issue_changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueHistory{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUE_HISTORY_TABLE}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueHistory + +func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + historyIdGen := didgen.NewDomainIdGenerator(&models.LinearIssueHistory{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueHistory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + event := inputRow.(*models.LinearIssueHistory) + changelog := &ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{Id: historyIdGen.Generate(connectionId, event.Id)}, + IssueId: issueIdGen.Generate(connectionId, event.IssueId), + FieldId: "state", + FieldName: "status", + OriginalFromValue: event.FromStateName, + OriginalToValue: event.ToStateName, + CreatedDate: event.CreatedAt, + } + if event.FromStateType != "" { + changelog.FromValue = StatusFromStateType(event.FromStateType) + } + if event.ToStateType != "" { + changelog.ToValue = StatusFromStateType(event.ToStateType) + } + if event.ActorId != "" { + changelog.AuthorId = accountIdGen.Generate(connectionId, event.ActorId) + } + return []interface{}{changelog}, nil + }, + }) + if err != nil { + return err + } + if err := converter.Execute(); err != nil { + return err + } + + return deriveLeadTimeFromHistory(db, connectionId, data.Options.TeamId, issueIdGen) +} + +// deriveLeadTimeFromHistory refines each issue's lead time from its recorded +// state transitions: the span from the issue's first transition into an +// in-progress state to its first transition into a done state thereafter (the +// active cycle time). This is the value that genuinely requires history and is +// more accurate than the coarse createdAt -> resolutionDate fallback set by +// ConvertIssues, so it overrides that fallback when the transitions exist. +// Issues whose history lacks an in-progress -> done sequence keep the fallback. +func deriveLeadTimeFromHistory(db dal.Dal, connectionId uint64, teamId string, issueIdGen *didgen.DomainIdGenerator) errors.Error { + var events []models.LinearIssueHistory + if err := db.All(&events, + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, teamId), + dal.Orderby("h.issue_id, h.created_at"), + ); err != nil { + return err + } + + type leadWindow struct { + startedAt *time.Time + doneAt *time.Time + } + windows := map[string]*leadWindow{} + for i := range events { + event := events[i] + window := windows[event.IssueId] + if window == nil { + window = &leadWindow{} + windows[event.IssueId] = window + } + switch StatusFromStateType(event.ToStateType) { + case ticket.IN_PROGRESS: + if window.startedAt == nil { + createdAt := event.CreatedAt + window.startedAt = &createdAt + } + case ticket.DONE: + if window.startedAt != nil && window.doneAt == nil { + createdAt := event.CreatedAt + window.doneAt = &createdAt + } + } + } + + for issueId, window := range windows { + if window.startedAt == nil || window.doneAt == nil || !window.doneAt.After(*window.startedAt) { + continue + } + minutes := uint(window.doneAt.Sub(*window.startedAt).Minutes()) + if err := db.UpdateColumn( + &ticket.Issue{}, "lead_time_minutes", minutes, + dal.Where("id = ?", issueIdGen.Generate(connectionId, issueId)), + ); err != nil { + return err + } + } + return nil +} diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go new file mode 100644 index 00000000000..006b92f9701 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Extract Issue History", + EntryPoint: ExtractIssueHistory, + EnabledByDefault: true, + Description: "Extract raw issue history into tool layer table _tool_linear_issue_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssueHistory + +func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiEvent := &GraphqlQueryHistory{} + if err := errors.Convert(json.Unmarshal(row.Data, apiEvent)); err != nil { + return nil, err + } + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + event := &models.LinearIssueHistory{ + ConnectionId: data.Options.ConnectionId, + Id: apiEvent.Id, + IssueId: issueRef.OwningIssueId(), + CreatedAt: apiEvent.CreatedAt, + } + if apiEvent.Actor != nil { + event.ActorId = apiEvent.Actor.Id + } + if apiEvent.FromState != nil { + event.FromStateId = apiEvent.FromState.Id + event.FromStateName = apiEvent.FromState.Name + event.FromStateType = apiEvent.FromState.Type + } + if apiEvent.ToState != nil { + event.ToStateId = apiEvent.ToState.Id + event.ToStateName = apiEvent.ToState.Name + event.ToStateType = apiEvent.ToState.Type + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/label_convertor.go b/backend/plugins/linear/tasks/label_convertor.go new file mode 100644 index 00000000000..e94826d4775 --- /dev/null +++ b/backend/plugins/linear/tasks/label_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ + Name: "Convert Issue Labels", + EntryPoint: ConvertIssueLabels, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_labels into domain layer table issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueLabel{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.IssueLabel{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueLabels + +func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + + cursor, err := db.Cursor( + dal.Select("l.*"), + dal.From("_tool_linear_issue_labels l"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = l.connection_id AND i.id = l.issue_id)"), + dal.Where("l.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueLabel{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + label := inputRow.(*models.LinearIssueLabel) + domainLabel := &ticket.IssueLabel{ + IssueId: issueIdGen.Generate(connectionId, label.IssueId), + LabelName: label.LabelName, + } + return []interface{}{domainLabel}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go new file mode 100644 index 00000000000..0a8cfe9da7f --- /dev/null +++ b/backend/plugins/linear/tasks/shared.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// issuesToCollectChildrenClauses builds the cursor clauses that drive per-issue +// child collection (comments, history). When `since` is non-nil (an incremental +// run), it restricts the sweep to issues updated since the last successful +// collection, so unchanged issues no longer trigger a request every run. On a +// full sync `since` is nil and all of the team's issues are swept. +func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *time.Time) []dal.Clause { + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + } + if since != nil { + clauses = append(clauses, dal.Where("updated_at > ?", *since)) + } + return clauses +} + +// priorityLabels maps Linear's integer priority to its human-readable label. +// Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. +var priorityLabels = map[int]string{ + 0: "No priority", + 1: "Urgent", + 2: "High", + 3: "Medium", + 4: "Low", +} + +// PriorityLabel returns the human-readable label for a Linear priority value. +func PriorityLabel(priority int) string { + if label, ok := priorityLabels[priority]; ok { + return label + } + return "No priority" +} + +// StatusFromStateType maps a Linear WorkflowState.type to a DevLake standard +// issue status. Linear's state types are standardized, so no user-supplied +// mapping is required: +// +// triage, backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +// +// "triage" is the inbox state issues land in before they are accepted into a +// workflow; it is treated as not-yet-started (TODO). Any unrecognized type +// falls back to OTHER so unexpected API values surface rather than silently +// masquerading as a known status. +func StatusFromStateType(stateType string) string { + switch stateType { + case "triage", "backlog", "unstarted": + return ticket.TODO + case "started": + return ticket.IN_PROGRESS + case "completed", "canceled": + return ticket.DONE + default: + return ticket.OTHER + } +} diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go new file mode 100644 index 00000000000..ad2fc7bc6d0 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestIssuesToCollectChildrenClauses pins the incremental behaviour of the +// per-issue child collectors (comments, history): a full sync sweeps every +// issue, while an incremental run adds an updated_at filter so unchanged issues +// are skipped instead of triggering a request each run. +func TestIssuesToCollectChildrenClauses(t *testing.T) { + // full sync: no `since` -> select/from/where(connection,team) only + full := issuesToCollectChildrenClauses(1, "team-1", nil) + assert.Len(t, full, 3) + + // incremental: a `since` adds the updated_at filter clause + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental := issuesToCollectChildrenClauses(1, "team-1", &since) + assert.Len(t, incremental, 4) +} diff --git a/backend/plugins/linear/tasks/shared_test.go b/backend/plugins/linear/tasks/shared_test.go new file mode 100644 index 00000000000..eaf737c9022 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/stretchr/testify/assert" +) + +// TestStatusFromStateType pins the mapping for every Linear WorkflowState.type +// value. Linear's state types are standardized; "triage" is the inbox state +// issues land in before they are accepted, so it maps to TODO. Any genuinely +// unknown type falls back to OTHER. +func TestStatusFromStateType(t *testing.T) { + cases := map[string]string{ + "backlog": ticket.TODO, + "unstarted": ticket.TODO, + "triage": ticket.TODO, + "started": ticket.IN_PROGRESS, + "completed": ticket.DONE, + "canceled": ticket.DONE, + "": ticket.OTHER, + "something": ticket.OTHER, + } + for stateType, want := range cases { + assert.Equal(t, want, StatusFromStateType(stateType), "state type %q", stateType) + } +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go new file mode 100644 index 00000000000..7ebe467a908 --- /dev/null +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertSprintIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Sprint Issues", + EntryPoint: ConvertSprintIssues, + EnabledByDefault: true, + Description: "Link issues to their cycle (sprint) in the domain layer table sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.SprintIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprintIssues + +func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + + // Sprint membership is derived from each issue's cycle_id. Clear this team's + // existing sprint_issues up front so issues that have since left their cycle + // leave no stale rows: the batch divider only deletes outdated records when + // it produces at least one row of the type, which misses the case where + // every issue has been removed from its cycle. + var teamIssues []models.LinearIssue + if err := db.All(&teamIssues, + dal.Select("id"), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ); err != nil { + return err + } + if len(teamIssues) > 0 { + issueIds := make([]string, len(teamIssues)) + for i, issue := range teamIssues { + issueIds[i] = issueIdGen.Generate(connectionId, issue.Id) + } + if err := db.Delete(&ticket.SprintIssue{}, dal.Where("issue_id IN ?", issueIds)); err != nil { + return err + } + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + sprintIssue := &ticket.SprintIssue{ + SprintId: cycleIdGen.Generate(connectionId, issue.CycleId), + IssueId: issueIdGen.Generate(connectionId, issue.Id), + } + return []interface{}{sprintIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/task_data.go b/backend/plugins/linear/tasks/task_data.go new file mode 100644 index 00000000000..19f991d5f75 --- /dev/null +++ b/backend/plugins/linear/tasks/task_data.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearOptions are the per-scope options passed to a pipeline task. +type LinearOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// LinearTaskData is the shared context handed to every Linear subtask. +type LinearTaskData struct { + Options *LinearOptions + GraphqlClient *api.GraphqlAsyncClient + TimeAfter *time.Time + // ScopeConfig carries the resolved scope config (e.g. label-based issue-type + // mapping). Never nil: PrepareTaskData defaults it to an empty config. + ScopeConfig *models.LinearScopeConfig +} + +type LinearApiParams models.LinearApiParams diff --git a/backend/plugins/linear/tasks/workflow_state_collector.go b/backend/plugins/linear/tasks/workflow_state_collector.go new file mode 100644 index 00000000000..94c22d1d3a5 --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_WORKFLOW_STATES_TABLE = "linear_workflow_states" + +// GraphqlQueryWorkflowStateWrapper is the team-scoped paginated `states` query. +type GraphqlQueryWorkflowStateWrapper struct { + Team struct { + States struct { + Nodes []GraphqlQueryWorkflowState + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"states(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryWorkflowState struct { + Id string + Name string + Type string + Color string + Position float64 +} + +var CollectWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Collect Workflow States", + EntryPoint: CollectWorkflowStates, + EnabledByDefault: true, + Description: "Collect workflow states for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectWorkflowStates + +func CollectWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryWorkflowStateWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryWorkflowStateWrapper) + return query.Team.States.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryWorkflowStateWrapper) + for _, state := range query.Team.States.Nodes { + messages = append(messages, errors.Must1(json.Marshal(state))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/workflow_state_extractor.go b/backend/plugins/linear/tasks/workflow_state_extractor.go new file mode 100644 index 00000000000..75a2099aeaf --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_extractor.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Extract Workflow States", + EntryPoint: ExtractWorkflowStates, + EnabledByDefault: true, + Description: "Extract raw workflow state data into tool layer table _tool_linear_workflow_states", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractWorkflowStates + +func ExtractWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiState := &GraphqlQueryWorkflowState{} + if err := errors.Convert(json.Unmarshal(row.Data, apiState)); err != nil { + return nil, err + } + state := &models.LinearWorkflowState{ + ConnectionId: data.Options.ConnectionId, + Id: apiState.Id, + TeamId: data.Options.TeamId, + Name: apiState.Name, + Type: apiState.Type, + Color: apiState.Color, + Position: apiState.Position, + } + return []interface{}{state}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 686b99816b0..0d4482ca6c9 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -28,8 +28,8 @@ import ( bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" - claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" @@ -44,6 +44,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" @@ -88,6 +89,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) checker.FeedIn("jenkins/models", jenkins.Jenkins{}.GetTablesInfo) checker.FeedIn("jira/models", jira.Jira{}.GetTablesInfo) + checker.FeedIn("linear/models", linear.Linear{}.GetTablesInfo) checker.FeedIn("org", org.Org{}.GetTablesInfo) checker.FeedIn("pagerduty/models", pagerduty.PagerDuty{}.GetTablesInfo) checker.FeedIn("refdiff/models", refdiff.RefDiff{}.GetTablesInfo) diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index b8bd6672f41..d1fff9c79e2 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -31,6 +31,7 @@ import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; @@ -59,6 +60,7 @@ export const pluginConfigs: IPluginConfig[] = [ GitLabConfig, JenkinsConfig, JiraConfig, + LinearConfig, PagerDutyConfig, RootlyConfig, SlackConfig, diff --git a/config-ui/src/plugins/register/linear/assets/icon.svg b/config-ui/src/plugins/register/linear/assets/icon.svg new file mode 100644 index 00000000000..64cddfdea54 --- /dev/null +++ b/config-ui/src/plugins/register/linear/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/linear/config.tsx b/config-ui/src/plugins/register/linear/config.tsx new file mode 100644 index 00000000000..28cf9f98a95 --- /dev/null +++ b/config-ui/src/plugins/register/linear/config.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const LinearConfig: IPluginConfig = { + plugin: 'linear', + name: 'Linear', + icon: ({ color }) => , + sort: 13, + connection: { + docLink: 'https://developers.linear.app/docs', + initialValues: { + endpoint: 'https://api.linear.app/graphql', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Linear GraphQL API base URL.', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'Your Linear personal API key (Settings → Security & access → Personal API keys).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (1500).', + defaultValue: 1500, + }, + ], + }, + dataScope: { + title: 'Teams', + searchPlaceholder: 'Search teams...', + }, +}; diff --git a/config-ui/src/plugins/register/linear/index.ts b/config-ui/src/plugins/register/linear/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/linear/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index e820f72cdd4..34c6ed63b14 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -45,6 +45,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.name}`; case 'asana': return `${scope.gid}`; + case 'linear': + return `${scope.teamId}`; default: return `${scope.id}`; } diff --git a/grafana/dashboards/Linear.json b/grafana/dashboards/Linear.json new file mode 100644 index 00000000000..1e748264206 --- /dev/null +++ b/grafana/dashboards/Linear.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Linear", + "url": "https://devlake.apache.org/docs/Configuration/Linear" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Linear.\n- Data Source Required: Linear", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "mysql", + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "9.5.15", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'linear%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'linear%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Linear", + "uid": "linear-dashboard", + "version": 1, + "weekStart": "" +} From 1b0d5800b4a628a53691e925e5c3a49a02e7c2c8 Mon Sep 17 00:00:00 2001 From: bujjibabukatta Date: Fri, 12 Jun 2026 19:22:21 +0530 Subject: [PATCH 09/10] fix(ui): guard against empty plugin prop on connection creation page (#8918) --- config-ui/src/plugins/components/connection-form/index.tsx | 6 +++++- config-ui/src/routes/connection/connections.tsx | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config-ui/src/plugins/components/connection-form/index.tsx b/config-ui/src/plugins/components/connection-form/index.tsx index 26432cee70f..b61c1d5d076 100644 --- a/config-ui/src/plugins/components/connection-form/index.tsx +++ b/config-ui/src/plugins/components/connection-form/index.tsx @@ -110,7 +110,7 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { const { name, connection: { docLink, fields, initialValues }, - } = getPluginConfig(plugin); + } = getPluginConfig(plugin) ?? {}; const disabled = useMemo(() => { return Object.values(errors).some(Boolean); @@ -118,6 +118,10 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { const sanitizedCustomHeaders = useMemo(() => sanitizeCustomHeaders(values.customHeaders), [values.customHeaders]); + if (!plugin || !name) { + return null; + } + const handleTest = async () => { const isUpdate = type === 'update' && !!connectionId; await operator( diff --git a/config-ui/src/routes/connection/connections.tsx b/config-ui/src/routes/connection/connections.tsx index 62427c45ccd..87b358b9ee9 100644 --- a/config-ui/src/routes/connection/connections.tsx +++ b/config-ui/src/routes/connection/connections.tsx @@ -61,8 +61,9 @@ export const Connections = () => { setPlugin(plugin); }; - const handleShowFormDialog = () => { + const handleShowFormDialog = (pluginName?: string) => { setType('form'); + if (pluginName) setPlugin(pluginName); }; const handleHideDialog = () => { @@ -168,7 +169,7 @@ export const Connections = () => { )} - {type === 'form' && pluginConfig && ( + {type === 'form' && plugin && pluginConfig && ( Date: Fri, 12 Jun 2026 19:27:03 +0530 Subject: [PATCH 10/10] fix(jira): rename "Jira Server" to "Jira Data Center" in UI labels (#8903) * fix(jira): rename Jira Server to Jira Data Center in UI labels * fix(jira): show both Jira Server and Jira Data Center labels --- backend/plugins/webhook/api/issues.go | 19 ++++--------------- .../src/plugins/register/jira/config.tsx | 2 +- .../register/jira/connection-fields/auth.tsx | 4 ++-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index ae6919b7bfa..9487cf6877a 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -19,11 +19,12 @@ package api import ( "fmt" - "github.com/apache/incubator-devlake/core/log" - "github.com/apache/incubator-devlake/helpers/dbhelper" "net/http" "time" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/helpers/dbhelper" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -283,7 +284,7 @@ func CloseIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput err := connectionHelper.FirstByName(connection, input.Params) return closeIssue(input, err, connection) } - + // CloseIssueByBodyByName // @Summary close an issue by connection name (body-based) // @Description Close an incident using connection name + issueKey in request body. @@ -333,12 +334,6 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo if domainIssue.ResolutionDate == nil { domainIssue.ResolutionDate = &now } - if domainIssue.LeadTimeMinutes == nil || *domainIssue.LeadTimeMinutes == 0 { - if domainIssue.CreatedDate != nil { - temp := uint(domainIssue.ResolutionDate.Sub(*domainIssue.CreatedDate).Minutes()) - domainIssue.LeadTimeMinutes = &temp - } - } // save err = tx.Update(domainIssue) if err != nil { @@ -355,12 +350,6 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo if domainIncident.ResolutionDate == nil { domainIncident.ResolutionDate = &now } - if domainIncident.LeadTimeMinutes == nil || *domainIncident.LeadTimeMinutes == 0 { - if domainIncident.CreatedDate != nil { - temp := uint(domainIncident.ResolutionDate.Sub(*domainIncident.CreatedDate).Minutes()) - domainIncident.LeadTimeMinutes = &temp - } - } // save err = tx.Update(domainIncident) if err != nil { diff --git a/config-ui/src/plugins/register/jira/config.tsx b/config-ui/src/plugins/register/jira/config.tsx index c6544d6e9e7..d82186577d5 100644 --- a/config-ui/src/plugins/register/jira/config.tsx +++ b/config-ui/src/plugins/register/jira/config.tsx @@ -49,7 +49,7 @@ export const JiraConfig: IPluginConfig = { 'By default, DevLake uses dynamic rate limit for optimized data collection for Jira. But you can adjust the collection speed by setting up your desirable rate limit.', learnMore: DOC_URL.PLUGIN.JIRA.RATE_LIMIT, externalInfo: - 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server, please contact your admin for more information.', + 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server / Jira Data Center, please contact your admin for more information.', defaultValue: 10000, }, ], diff --git a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx index a85999ba3b0..bc43a6acec3 100644 --- a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx +++ b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx @@ -124,7 +124,7 @@ export const Auth = ({ type, initialValues, values, setValues, setErrors }: Prop Jira Cloud - Jira Server + Jira Server / Jira Data Center }