Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions backend/core/plugin/plugin_blueprint.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,17 @@ type DataSourcePluginBlueprintV200 interface {
) (models.PipelinePlan, []Scope, errors.Error)
}

// DataSourcePluginBlueprintV200WithProjectName extends V200 to accept the
// projectName from the blueprint generation flow so plugins can populate
// user-definable fields (e.g. `ProjectName` on boards) when producing scopes.
type DataSourcePluginBlueprintV200WithProjectName interface {
MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*models.BlueprintScope,
projectName string,
) (models.PipelinePlan, []Scope, errors.Error)
}

// BlueprintConnectionV200 contains the pluginName/connectionId and related Scopes,

// MetricPluginBlueprintV200 is similar to the DataSourcePluginBlueprintV200
Expand Down
12 changes: 10 additions & 2 deletions backend/plugins/jira/api/blueprint_v200.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func MakeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
projectName string,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
// load connection, scope and scopeConfig from the db
connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
Expand All @@ -54,11 +55,11 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}

plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection, projectName)
if err != nil {
return nil, nil, err
}
scopes, err := makeScopesV200(scopeDetails, connection)
scopes, err := makeScopesV200(scopeDetails, connection, projectName)
if err != nil {
return nil, nil, err
}
Expand All @@ -70,6 +71,7 @@ func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
scopeDetails []*srvhelper.ScopeDetail[models.JiraBoard, models.JiraScopeConfig],
connection *models.JiraConnection,
projectName string,
) (coreModels.PipelinePlan, errors.Error) {
plan := make(coreModels.PipelinePlan, len(scopeDetails))
for i, scopeDetail := range scopeDetails {
Expand Down Expand Up @@ -103,10 +105,16 @@ func makeDataSourcePipelinePlanV200(
func makeScopesV200(
scopeDetails []*srvhelper.ScopeDetail[models.JiraBoard, models.JiraScopeConfig],
connection *models.JiraConnection,
projectName string,
) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, scopeDetail := range scopeDetails {
jiraBoard, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
// populate the tool-layer board's ProjectName from blueprint input so
// downstream code and templates can rely on it
if jiraBoard != nil {
jiraBoard.ProjectName = projectName
}
// add board to scopes
if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
domainBoard := &ticket.Board{
Expand Down
10 changes: 8 additions & 2 deletions backend/plugins/jira/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"

"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/api"
Expand Down Expand Up @@ -195,8 +196,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
return nil, errors.Default.Wrap(err, "failed to create jira api client")
}

var scope *models.JiraBoard
if op.BoardId != 0 {
var scope *models.JiraBoard
// support v100 & advance mode
// If we still cannot find the record in db, we have to request from remote server and save it to db
db := taskCtx.GetDal()
Expand Down Expand Up @@ -249,16 +250,21 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int
Options: &op,
ApiClient: jiraApiClient,
JiraServerInfo: *info,
Board: scope,
}

// Board is set above; any project-name mapping should be provided by the
// blueprint/Scope data (user-definable `ProjectName` column on the board).

return taskData, nil
}

func (p Jira) MakeDataSourcePipelinePlanV200(
connectionId uint64,
scopes []*coreModels.BlueprintScope,
projectName string,
) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) {
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, projectName)
}

func (p Jira) RootPkgPath() string {
Expand Down
14 changes: 9 additions & 5 deletions backend/plugins/jira/models/board.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,15 @@ type JiraBoard struct {
common.Scope `mapstructure:",squash"`
BoardId uint64 `json:"boardId" mapstructure:"boardId" validate:"required" gorm:"primaryKey"`
ProjectId uint `json:"projectId" mapstructure:"projectId"`
Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"`
Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"`
Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"`
Jql string `json:"jql" mapstructure:"jql"`
SubQuery string `json:"subQuery" mapstructure:"subQuery"`
// ProjectName is a user-definable field that can be set by the blueprint
// and used when composing ExtraJQL. It represents the DevLake project
// name (or other identifier) associated with this board.
ProjectName string `json:"projectName" mapstructure:"projectName" gorm:"type:varchar(255)"`
Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"`
Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"`
Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"`
Jql string `json:"jql" mapstructure:"jql"`
SubQuery string `json:"subQuery" mapstructure:"subQuery"`
}

func (b JiraBoard) ScopeId() string {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)

type JiraScopeConfig20260702 struct {
ExtraJQL string `gorm:"type:varchar(255)"`
}

func (JiraScopeConfig20260702) TableName() string {
return "_tool_jira_scope_configs"
}

type addExtraJQLToScopeConfig struct{}

func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(basicRes, &JiraScopeConfig20260702{})
}

func (*addExtraJQLToScopeConfig) Version() uint64 {
return 20260702000000
}

func (*addExtraJQLToScopeConfig) Name() string {
return "add extra_jql to _tool_jira_scope_configs"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)

type JiraBoardProjectName20260704 struct {
ProjectName string `gorm:"type:varchar(255)"`
}

func (JiraBoardProjectName20260704) TableName() string {
return "_tool_jira_boards"
}

type addProjectNameToBoards20260704 struct{}

func (script *addProjectNameToBoards20260704) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(basicRes, &JiraBoardProjectName20260704{})
}

func (*addProjectNameToBoards20260704) Version() uint64 {
return 20260704000000
}

func (*addProjectNameToBoards20260704) Name() string {
return "add project_name to _tool_jira_boards"
}
2 changes: 2 additions & 0 deletions backend/plugins/jira/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,5 +56,7 @@ func All() []plugin.MigrationScript {
new(updateScopeConfig),
new(addFixVersions20250619),
new(addSubQueryToBoards),
new(addProjectNameToBoards20260704),
new(addExtraJQLToScopeConfig),
}
}
7 changes: 7 additions & 0 deletions backend/plugins/jira/models/scope_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package models

import (
"regexp"
"text/template"

"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
Expand Down Expand Up @@ -49,6 +50,7 @@ type JiraScopeConfig struct {
TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"`
ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"`
DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"`
ExtraJQL string `mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"`
}

func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId uint64) {
Expand All @@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error {
return errors.Convert(err)
}
}
if r.ExtraJQL != "" {
if _, tmplErr := template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL); tmplErr != nil {
return errors.BadInput.Wrap(errors.Convert(tmplErr), "invalid ExtraJQL template")
}
}
return nil
}

Expand Down
89 changes: 77 additions & 12 deletions backend/plugins/jira/tasks/issue_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ limitations under the License.
package tasks

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"text/template"
"time"

"github.com/apache/incubator-devlake/core/dal"
Expand Down Expand Up @@ -77,7 +79,15 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
// The board Agile API applies kanban sub-filters server-side, which silently
// excludes resolved issues (e.g. those with a released fixVersion).
// The search API with the saved filter JQL returns all matching issues.
filterJql := buildFilterJQL(data.FilterId, incrementalJql)
var extraJql string
if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL != "" {
renderedJql, renderErr := renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data)
if renderErr != nil {
return renderErr
}
extraJql = renderedJql
}
filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql)
logger.Info("collecting issues via search API with JQL: %s", filterJql)

pageSize := data.Options.PageSize
Expand All @@ -99,19 +109,74 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
return apiCollector.Execute()
}

func buildFilterJQL(filterId string, incrementalJql string) string {
if filterId == "" {
return incrementalJql
// JqlTemplateData holds the variables available inside an ExtraJQL template.
// Users reference these with Go template syntax, e.g. `{{.BoardName}}`.
type JqlTemplateData struct {
BoardId uint64 // numeric ID of the connected Jira board
BoardName string // display name of the connected Jira board
DevLakeProjectName string // name of the DevLake project this board belongs to
}

// renderExtraJQL executes the ExtraJQL scope-config field as a Go text/template,
// substituting board-level variables so the same scope config can produce
// different JQL for different boards.
//
// The template is parsed with an empty FuncMap (no built-in helpers such as
// printf) and missingkey=error so that typos in variable names produce an
// explicit error rather than silently rendering "<no value>".
func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error) {
tmpl, err := template.New("extraJql").
Funcs(template.FuncMap{}).
Option("missingkey=error").
Parse(tmplStr)
if err != nil {
return "", errors.BadInput.Wrap(err, "invalid ExtraJQL template")
}
// Use Jira's `filter = {id}` syntax to reference the saved filter.
// This avoids parenthesization bugs when composing raw JQL strings
// that may contain OR/AND operators.
if incrementalJql == "ORDER BY created ASC" {
return fmt.Sprintf("filter = %s ORDER BY created ASC", filterId)

vars := JqlTemplateData{
BoardId: data.Options.BoardId,
DevLakeProjectName: "",
}
if data.Board != nil {
vars.BoardName = data.Board.Name
vars.DevLakeProjectName = data.Board.ProjectName
}

var buf bytes.Buffer
if execErr := tmpl.Execute(&buf, vars); execErr != nil {
return "", errors.BadInput.Wrap(execErr, "failed to render ExtraJQL template")
}
return buf.String(), nil
}

// buildFilterJQL composes a final JQL query from three inputs:
// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`)
// - extraJql: optional user-supplied JQL fragment appended as an AND condition
// (e.g. `project = "MyComponent"`) to scope a large board down to one project
// - incrementalJql: the time-based clause generated by buildJQL, always ending
// with "ORDER BY created ASC"
//
// extraJql is wrapped in parentheses so that any OR/NOT operators inside it
// do not interfere with the surrounding AND chain.
func buildFilterJQL(filterId string, extraJql string, incrementalJql string) string {
const orderBy = "ORDER BY created ASC"

var conditions []string
if filterId != "" {
conditions = append(conditions, fmt.Sprintf("filter = %s", filterId))
}
if extraJql != "" {
conditions = append(conditions, fmt.Sprintf("(%s)", extraJql))
}
if incrementalJql != orderBy {
// strip the trailing " ORDER BY created ASC" to isolate the time condition
conditions = append(conditions, strings.TrimSuffix(incrementalJql, " "+orderBy))
}

if len(conditions) == 0 {
return orderBy
}
// incrementalJql contains "updated >= '...' ORDER BY created ASC"
// We need to insert the filter reference before the incremental clause
return fmt.Sprintf("filter = %s AND %s", filterId, incrementalJql)
return strings.Join(conditions, " AND ") + " " + orderBy
}

func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int) errors.Error {
Expand Down
Loading