Skip to content

Add open/closed field support for issue index #25708

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 7, 2023
Merged

Add open/closed field support for issue index #25708

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d414d28
Add open/closed field support for issue index
techknowlogick Jul 5, 2023
714b0a4
make fmt
techknowlogick Jul 5, 2023
e181acd
Merge branch 'main' into update-meili-indexer
techknowlogick Jul 7, 2023
5e4807d
Add issue type to search index
techknowlogick Jul 7, 2023
ff89436
fix syntax
techknowlogick Jul 7, 2023
2e08ce5
Merge branch 'main' into update-meili-indexer
GiteaBot Jul 7, 2023
8de8e12
Merge branch 'main' into update-meili-indexer
GiteaBot Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/indexer/issues/bleve/bleve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error {

// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
var repoQueriesP []*query.NumericRangeQuery
for _, repoID := range repoIDs {
repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "repo_id"))
Expand Down
2 changes: 1 addition & 1 deletion modules/indexer/issues/bleve/bleve_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func TestBleveIndexAndSearch(t *testing.T) {
}

for _, kw := range keywords {
res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0)
res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0, "")
assert.NoError(t, err)

ids := make([]int64, 0, len(res.Hits))
Expand Down
2 changes: 1 addition & 1 deletion modules/indexer/issues/db/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error {
}

// Search searches for issues
func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
if err != nil {
return nil, err
Expand Down
2 changes: 1 addition & 1 deletion modules/indexer/issues/elasticsearch/elasticsearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {

// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
query := elastic.NewBoolQuery()
query = query.Must(kwQuery)
Expand Down
20 changes: 13 additions & 7 deletions modules/indexer/issues/indexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,18 @@ func UpdateIssueIndexer(issue *issues_model.Issue) {
comments = append(comments, comment.Content)
}
}
issueType := "issue"
if issue.IsPull {
issueType = "pull"
}
indexerData := &internal.IndexerData{
ID: issue.ID,
RepoID: issue.RepoID,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
ID: issue.ID,
RepoID: issue.RepoID,
State: string(issue.State()),
IssueType: issueType,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
}
log.Debug("Adding to channel: %v", indexerData)
if err := issueIndexerQueue.Push(indexerData); err != nil {
Expand Down Expand Up @@ -278,10 +284,10 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {

// SearchIssuesByKeyword search issue ids by keywords and repo id
// WARNNING: You have to ensure user have permission to visit repoIDs' issues
func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) {
func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword, state string) ([]int64, error) {
var issueIDs []int64
indexer := *globalIndexer.Load()
res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0)
res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0, state)
if err != nil {
return nil, err
}
Expand Down
16 changes: 8 additions & 8 deletions modules/indexer/issues/indexer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,19 +50,19 @@ func TestBleveSearchIssues(t *testing.T) {

time.Sleep(5 * time.Second)

ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{2}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "")
assert.NoError(t, err)
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)
}
Expand All @@ -73,19 +73,19 @@ func TestDBSearchIssues(t *testing.T) {
setting.Indexer.IssueType = "db"
InitIssueIndexer(true)

ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2")
ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{2}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "")
assert.NoError(t, err)
assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids)

ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good")
ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)
}
4 changes: 2 additions & 2 deletions modules/indexer/issues/internal/indexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type Indexer interface {
internal.Indexer
Index(ctx context.Context, issue []*IndexerData) error
Delete(ctx context.Context, ids ...int64) error
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error)
}

// NewDummyIndexer returns a dummy indexer
Expand All @@ -37,6 +37,6 @@ func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error {
return fmt.Errorf("indexer is not ready")
}

func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) {
return nil, fmt.Errorf("indexer is not ready")
}
16 changes: 9 additions & 7 deletions modules/indexer/issues/internal/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ package internal

// IndexerData data stored in the issue indexer
type IndexerData struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Title string `json:"title"`
Content string `json:"content"`
Comments []string `json:"comments"`
IsDelete bool `json:"is_delete"`
IDs []int64 `json:"ids"`
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
State string `json:"state"` // open, closed, all
IssueType string `json:"type"` // issue or pull
Title string `json:"title"`
Content string `json:"content"`
Comments []string `json:"comments"`
IsDelete bool `json:"is_delete"`
IDs []int64 `json:"ids"`
}

// Match represents on search result
Expand Down
11 changes: 9 additions & 2 deletions modules/indexer/issues/meilisearch/meilisearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
)

const (
issueIndexerLatestVersion = 0
issueIndexerLatestVersion = 1
)

var _ internal.Indexer = &Indexer{}
Expand Down Expand Up @@ -70,12 +70,19 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error {

// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) {
repoFilters := make([]string, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
}
filter := strings.Join(repoFilters, " OR ")
if state == "open" || state == "closed" {
if filter != "" {
filter = "(" + filter + ") AND state = " + state
} else {
filter = "state = " + state
}
}
searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{
Filter: filter,
Limit: int64(limit),
Expand Down
4 changes: 2 additions & 2 deletions routers/api/v1/repo/issue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ func SearchIssues(ctx *context.APIContext) {
}
var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return
}
Expand Down Expand Up @@ -394,7 +394,7 @@ func ListIssues(ctx *context.APIContext) {
var issueIDs []int64
var labelIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword)
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err)
return
Expand Down
6 changes: 3 additions & 3 deletions routers/web/repo/issue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti

var issueIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword)
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword, ctx.FormString("state"))
if err != nil {
if issue_indexer.IsAvailable(ctx) {
ctx.ServerError("issueIndexer.Search", err)
Expand Down Expand Up @@ -2466,7 +2466,7 @@ func SearchIssues(ctx *context.Context) {
}
var issueIDs []int64
if len(keyword) > 0 && len(repoIDs) > 0 {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil {
if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil {
ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error())
return
}
Expand Down Expand Up @@ -2614,7 +2614,7 @@ func ListIssues(ctx *context.Context) {
var issueIDs []int64
var labelIDs []int64
if len(keyword) > 0 {
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword)
issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state"))
if err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
Expand Down
2 changes: 1 addition & 1 deletion routers/web/user/home.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword
if err != nil {
return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err)
}
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword)
issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state"))
if err != nil {
return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err)
}
Expand Down