diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index 126e513e24d..f4da4df57d7 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -25,6 +25,8 @@ import ( "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/jira/api" @@ -195,8 +197,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int return nil, errors.Default.Wrap(err, "failed to create jira api client") } + var scope *models.JiraBoard if op.BoardId != 0 { - var scope *models.JiraBoard // support v100 & advance mode // If we still cannot find the record in db, we have to request from remote server and save it to db db := taskCtx.GetDal() @@ -249,6 +251,22 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int Options: &op, ApiClient: jiraApiClient, JiraServerInfo: *info, + Board: scope, + } + + // Look up the DevLake project this board belongs to via project_mapping. + // This is best-effort: if the board is not yet mapped (e.g. first run or + // manual trigger outside a blueprint) we leave DevLakeProjectName empty. + if scope != nil { + domainBoardId := didgen.NewDomainIdGenerator(&models.JiraBoard{}).Generate(scope.ConnectionId, scope.BoardId) + var pm crossdomain.ProjectMapping + if lookupErr := db.First(&pm, dal.Where( + "`table` = ? AND row_id = ?", "boards", domainBoardId, + )); lookupErr == nil { + taskData.DevLakeProjectName = pm.ProjectName + } else if !db.IsErrorNotFound(lookupErr) { + logger.Warn(lookupErr, "failed to look up project mapping for board %d", scope.BoardId) + } } return taskData, nil diff --git a/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go new file mode 100644 index 00000000000..34966a56525 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraScopeConfig20260702 struct { + ExtraJQL string `gorm:"type:varchar(255)"` +} + +func (JiraScopeConfig20260702) TableName() string { + return "_tool_jira_scope_configs" +} + +type addExtraJQLToScopeConfig struct{} + +func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraScopeConfig20260702{}) +} + +func (*addExtraJQLToScopeConfig) Version() uint64 { + return 20260702000000 +} + +func (*addExtraJQLToScopeConfig) Name() string { + return "add extra_jql to _tool_jira_scope_configs" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 37fc6a5f917..0ee1243dd7c 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -56,5 +56,6 @@ func All() []plugin.MigrationScript { new(updateScopeConfig), new(addFixVersions20250619), new(addSubQueryToBoards), + new(addExtraJQLToScopeConfig), } } diff --git a/backend/plugins/jira/models/scope_config.go b/backend/plugins/jira/models/scope_config.go index a8bd78a981b..5dc6d07fed9 100644 --- a/backend/plugins/jira/models/scope_config.go +++ b/backend/plugins/jira/models/scope_config.go @@ -19,6 +19,7 @@ package models import ( "regexp" + "text/template" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -49,6 +50,7 @@ type JiraScopeConfig struct { TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"` DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"` + ExtraJQL string `mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"` } func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId uint64) { @@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error { return errors.Convert(err) } } + if r.ExtraJQL != "" { + if _, tmplErr := template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL); tmplErr != nil { + return errors.BadInput.Wrap(errors.Convert(tmplErr), "invalid ExtraJQL template") + } + } return nil } diff --git a/backend/plugins/jira/tasks/issue_collector.go b/backend/plugins/jira/tasks/issue_collector.go index 28aa473ff80..8a0951aa82f 100644 --- a/backend/plugins/jira/tasks/issue_collector.go +++ b/backend/plugins/jira/tasks/issue_collector.go @@ -18,12 +18,14 @@ limitations under the License. package tasks import ( + "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" + "text/template" "time" "github.com/apache/incubator-devlake/core/dal" @@ -77,7 +79,15 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { // The board Agile API applies kanban sub-filters server-side, which silently // excludes resolved issues (e.g. those with a released fixVersion). // The search API with the saved filter JQL returns all matching issues. - filterJql := buildFilterJQL(data.FilterId, incrementalJql) + var extraJql string + if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL != "" { + renderedJql, renderErr := renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data) + if renderErr != nil { + return renderErr + } + extraJql = renderedJql + } + filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql) logger.Info("collecting issues via search API with JQL: %s", filterJql) pageSize := data.Options.PageSize @@ -99,19 +109,73 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { return apiCollector.Execute() } -func buildFilterJQL(filterId string, incrementalJql string) string { - if filterId == "" { - return incrementalJql +// JqlTemplateData holds the variables available inside an ExtraJQL template. +// Users reference these with Go template syntax, e.g. `{{.BoardName}}`. +type JqlTemplateData struct { + BoardId uint64 // numeric ID of the connected Jira board + BoardName string // display name of the connected Jira board + DevLakeProjectName string // name of the DevLake project this board belongs to +} + +// renderExtraJQL executes the ExtraJQL scope-config field as a Go text/template, +// substituting board-level variables so the same scope config can produce +// different JQL for different boards. +// +// The template is parsed with an empty FuncMap (no built-in helpers such as +// printf) and missingkey=error so that typos in variable names produce an +// explicit error rather than silently rendering "". +func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error) { + tmpl, err := template.New("extraJql"). + Funcs(template.FuncMap{}). + Option("missingkey=error"). + Parse(tmplStr) + if err != nil { + return "", errors.BadInput.Wrap(err, "invalid ExtraJQL template") } - // Use Jira's `filter = {id}` syntax to reference the saved filter. - // This avoids parenthesization bugs when composing raw JQL strings - // that may contain OR/AND operators. - if incrementalJql == "ORDER BY created ASC" { - return fmt.Sprintf("filter = %s ORDER BY created ASC", filterId) + + vars := JqlTemplateData{ + BoardId: data.Options.BoardId, + DevLakeProjectName: data.DevLakeProjectName, + } + if data.Board != nil { + vars.BoardName = data.Board.Name + } + + var buf bytes.Buffer + if execErr := tmpl.Execute(&buf, vars); execErr != nil { + return "", errors.BadInput.Wrap(execErr, "failed to render ExtraJQL template") + } + return buf.String(), nil +} + +// buildFilterJQL composes a final JQL query from three inputs: +// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`) +// - extraJql: optional user-supplied JQL fragment appended as an AND condition +// (e.g. `project = "MyComponent"`) to scope a large board down to one project +// - incrementalJql: the time-based clause generated by buildJQL, always ending +// with "ORDER BY created ASC" +// +// extraJql is wrapped in parentheses so that any OR/NOT operators inside it +// do not interfere with the surrounding AND chain. +func buildFilterJQL(filterId string, extraJql string, incrementalJql string) string { + const orderBy = "ORDER BY created ASC" + + var conditions []string + if filterId != "" { + conditions = append(conditions, fmt.Sprintf("filter = %s", filterId)) + } + if extraJql != "" { + conditions = append(conditions, fmt.Sprintf("(%s)", extraJql)) + } + if incrementalJql != orderBy { + // strip the trailing " ORDER BY created ASC" to isolate the time condition + conditions = append(conditions, strings.TrimSuffix(incrementalJql, " "+orderBy)) + } + + if len(conditions) == 0 { + return orderBy } - // incrementalJql contains "updated >= '...' ORDER BY created ASC" - // We need to insert the filter reference before the incremental clause - return fmt.Sprintf("filter = %s AND %s", filterId, incrementalJql) + return strings.Join(conditions, " AND ") + " " + orderBy } func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int) errors.Error { diff --git a/backend/plugins/jira/tasks/issue_collector_test.go b/backend/plugins/jira/tasks/issue_collector_test.go index 7d5bdc1c1c9..4215923240b 100644 --- a/backend/plugins/jira/tasks/issue_collector_test.go +++ b/backend/plugins/jira/tasks/issue_collector_test.go @@ -20,6 +20,8 @@ package tasks import ( "testing" "time" + + "github.com/apache/incubator-devlake/plugins/jira/models" ) func Test_buildJQL(t *testing.T) { @@ -66,6 +68,7 @@ func Test_buildFilterJQL(t *testing.T) { tests := []struct { name string filterId string + extraJql string incrementalJql string want string }{ @@ -93,13 +96,122 @@ func Test_buildFilterJQL(t *testing.T) { incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", want: "updated >= '2024/01/01 00:00' ORDER BY created ASC", }, + { + name: "extra jql with filter full sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql with filter incremental sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql without filter", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `(project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql without filter, incremental sync", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `(project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql with OR operator is parenthesized", + filterId: "12345", + extraJql: `project = "A" OR project = "B"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "A" OR project = "B") ORDER BY created ASC`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := buildFilterJQL(tt.filterId, tt.incrementalJql); got != tt.want { + if got := buildFilterJQL(tt.filterId, tt.extraJql, tt.incrementalJql); got != tt.want { t.Errorf("buildFilterJQL() = %v, want %v", got, tt.want) } }) } } + +func Test_renderExtraJQL(t *testing.T) { + makeData := func(boardId uint64, boardName string, projectName string) *JiraTaskData { + return &JiraTaskData{ + Options: &JiraOptions{BoardId: boardId}, + Board: &models.JiraBoard{BoardId: boardId, Name: boardName}, + DevLakeProjectName: projectName, + } + } + + tests := []struct { + name string + tmpl string + data *JiraTaskData + want string + wantErr bool + }{ + { + name: "static JQL passes through unchanged", + tmpl: `project = "MyProject"`, + data: makeData(1, "My Board", ""), + want: `project = "MyProject"`, + }, + { + name: "BoardName substitution", + tmpl: `project = "{{.BoardName}}"`, + data: makeData(42, "Team Alpha", ""), + want: `project = "Team Alpha"`, + }, + { + name: "BoardId substitution", + tmpl: `cf[10001] = {{.BoardId}}`, + data: makeData(99, "Some Board", ""), + want: `cf[10001] = 99`, + }, + { + name: "DevLakeProjectName substitution", + tmpl: `component = "{{.DevLakeProjectName}}"`, + data: makeData(1, "Big Shared Board", "payments-service"), + want: `component = "payments-service"`, + }, + { + name: "nil Board falls back to empty BoardName", + tmpl: `project = "{{.BoardName}}"`, + data: &JiraTaskData{Options: &JiraOptions{BoardId: 1}, Board: nil}, + want: `project = ""`, + }, + { + name: "invalid template returns error", + tmpl: `project = "{{.Unclosed"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + { + name: "unknown field returns error (missingkey=error)", + tmpl: `component = "{{.Typo}}"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderExtraJQL(tt.tmpl, tt.data) + if (err != nil) != tt.wantErr { + t.Errorf("renderExtraJQL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got != tt.want { + t.Errorf("renderExtraJQL() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go index bfab9704f04..73758ee82fd 100644 --- a/backend/plugins/jira/tasks/task_data.go +++ b/backend/plugins/jira/tasks/task_data.go @@ -34,10 +34,12 @@ type JiraOptions struct { } type JiraTaskData struct { - Options *JiraOptions - ApiClient *api.ApiAsyncClient - JiraServerInfo models.JiraServerInfo - FilterId string + Options *JiraOptions + ApiClient *api.ApiAsyncClient + JiraServerInfo models.JiraServerInfo + FilterId string + Board *models.JiraBoard + DevLakeProjectName string } type JiraApiParams models.JiraApiParams diff --git a/config-ui/src/plugins/register/jira/config.tsx b/config-ui/src/plugins/register/jira/config.tsx index d82186577d5..bb4996e657e 100644 --- a/config-ui/src/plugins/register/jira/config.tsx +++ b/config-ui/src/plugins/register/jira/config.tsx @@ -64,6 +64,7 @@ export const JiraConfig: IPluginConfig = { typeMappings: {}, remotelinkCommitShaPattern: '', remotelinkRepoPattern: [], + extraJql: '', }, }, }; diff --git a/config-ui/src/plugins/register/jira/transformation.tsx b/config-ui/src/plugins/register/jira/transformation.tsx index 97cd20967b5..c9ad86bdb15 100644 --- a/config-ui/src/plugins/register/jira/transformation.tsx +++ b/config-ui/src/plugins/register/jira/transformation.tsx @@ -19,7 +19,7 @@ import { useState, useEffect } from 'react'; import { uniqWith } from 'lodash'; import { CaretRightOutlined } from '@ant-design/icons'; -import { theme, Collapse, Tag, Form, Select } from 'antd'; +import { theme, Collapse, Tag, Form, Select, Input } from 'antd'; import API from '@/api'; import { PageLoading, HelpTooltip, ExternalLink } from '@/components'; @@ -265,6 +265,26 @@ const renderCollapseItems = ({ } /> + + Extra JQL + + + } + extra='Tip: use Go template variables to make this dynamic, e.g. component = "{{.DevLakeProjectName}}"' + > + + onChangeTransformation({ + ...transformation, + extraJql: e.target.value, + }) + } + /> + ), },