store the foreign ID of issues during migration

models/fixtures/foreign_reference.yml

@@ -0,0 +1 @@
+[] # empty

models/foreignreference/error.go

@@ -0,0 +1,43 @@
+// Copyright 2022 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreignreference
+
+import (
+	"fmt"
+)
+
+// ErrLocalIndexNotExist represents a "LocalIndexNotExist" kind of error.
+type ErrLocalIndexNotExist struct {
+	RepoID       int64
+	ForeignIndex int64
+	Type         string
+}
+
+// ErrLocalIndexNotExist checks if an error is a ErrLocalIndexNotExist.
+func IsErrLocalIndexNotExist(err error) bool {
+	_, ok := err.(ErrLocalIndexNotExist)
+	return ok
+}
+
+func (err ErrLocalIndexNotExist) Error() string {
+	return fmt.Sprintf("repository %d has no LocalIndex for ForeignIndex %d of type %s", err.RepoID, err.ForeignIndex, err.Type)
+}
+
+// ErrForeignIndexNotExist represents a "ForeignIndexNotExist" kind of error.
+type ErrForeignIndexNotExist struct {
+	RepoID     int64
+	LocalIndex int64
+	Type       string
+}
+
+// ErrForeignIndexNotExist checks if an error is a ErrForeignIndexNotExist.
+func IsErrForeignIndexNotExist(err error) bool {
+	_, ok := err.(ErrForeignIndexNotExist)
+	return ok
+}
+
+func (err ErrForeignIndexNotExist) Error() string {
+	return fmt.Sprintf("repository %d has no ForeignIndex for LocalIndex %d of type %s", err.RepoID, err.LocalIndex, err.Type)
+}

models/foreignreference/foreignreference.go

@@ -0,0 +1,32 @@
+// Copyright 2022 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreignreference
+
+import (
+	"code.gitea.io/gitea/models/db"
+)
+
+// Type* are valid values for the Type field of ForeignReference
+const (
+	TypeIssue         = "issue"
+	TypePullRequest   = "pull_request"
+	TypeComment       = "comment"
+	TypeReview        = "review"
+	TypeReviewComment = "review_comment"
+	TypeRelease       = "release"
+)
+
+// ForeignReference represents external references
+type ForeignReference struct {
+	// RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type)
+	RepoID       int64  `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" `
+	LocalIndex   int64  `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID.
+	ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"`
+	Type         string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"`
+}
+
+func init() {
+	db.RegisterModel(new(ForeignReference))
+}

models/issue.go

@@ -15,6 +15,7 @@ import (
 
 	admin_model "code.gitea.io/gitea/models/admin"
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/foreignreference"
 	"code.gitea.io/gitea/models/issues"
 	"code.gitea.io/gitea/models/perm"
 	repo_model "code.gitea.io/gitea/models/repo"
@@ -67,11 +68,12 @@ type Issue struct {
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
 	ClosedUnix  timeutil.TimeStamp `xorm:"INDEX"`
 
-	Attachments      []*repo_model.Attachment `xorm:"-"`
-	Comments         []*Comment               `xorm:"-"`
-	Reactions        ReactionList             `xorm:"-"`
-	TotalTrackedTime int64                    `xorm:"-"`
-	Assignees        []*user_model.User       `xorm:"-"`
+	Attachments      []*repo_model.Attachment           `xorm:"-"`
+	Comments         []*Comment                         `xorm:"-"`
+	Reactions        ReactionList                       `xorm:"-"`
+	TotalTrackedTime int64                              `xorm:"-"`
+	Assignees        []*user_model.User                 `xorm:"-"`
+	ForeignReference *foreignreference.ForeignReference `xorm:"-"`
 
 	// IsLocked limits commenting abilities to users on an issue
 	// with write access
@@ -271,6 +273,29 @@ func (issue *Issue) loadReactions(ctx context.Context) (err error) {
 	return nil
 }
 
+func (issue *Issue) loadForeignReference(ctx context.Context) (err error) {
+	if issue.ForeignReference != nil {
+		return nil
+	}
+	reference := &foreignreference.ForeignReference{
+		RepoID:     issue.RepoID,
+		LocalIndex: issue.Index,
+		Type:       foreignreference.TypeIssue,
+	}
+	has, err := db.GetEngine(ctx).Get(reference)
+	if err != nil {
+		return err
+	} else if !has {
+		return foreignreference.ErrForeignIndexNotExist{
+			RepoID:     issue.RepoID,
+			LocalIndex: issue.Index,
+			Type:       foreignreference.TypeIssue,
+		}
+	}
+	issue.ForeignReference = reference
+	return nil
+}
+
 func (issue *Issue) loadMilestone(e db.Engine) (err error) {
 	if (issue.Milestone == nil || issue.Milestone.ID != issue.MilestoneID) && issue.MilestoneID > 0 {
 		issue.Milestone, err = getMilestoneByRepoID(e, issue.RepoID, issue.MilestoneID)
@@ -332,6 +357,10 @@ func (issue *Issue) loadAttributes(ctx context.Context) (err error) {
 		}
 	}
 
+	if err = issue.loadForeignReference(ctx); err != nil && !foreignreference.IsErrForeignIndexNotExist(err) {
+		return err
+	}
+
 	return issue.loadReactions(ctx)
 }
 
@@ -1110,6 +1139,26 @@ func GetIssueByIndex(repoID, index int64) (*Issue, error) {
 	return issue, nil
 }
 
+// GetIssueByForeignIndex returns raw issue by foreign ID
+func GetIssueByForeignIndex(ctx context.Context, repoID, foreignIndex int64) (*Issue, error) {
+	reference := &foreignreference.ForeignReference{
+		RepoID:       repoID,
+		ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+		Type:         foreignreference.TypeIssue,
+	}
+	has, err := db.GetEngine(ctx).Get(reference)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, foreignreference.ErrLocalIndexNotExist{
+			RepoID:       repoID,
+			ForeignIndex: foreignIndex,
+			Type:         foreignreference.TypeIssue,
+		}
+	}
+	return GetIssueByIndex(repoID, reference.LocalIndex)
+}
+
 // GetIssueWithAttrsByIndex returns issue by index in a repository.
 func GetIssueWithAttrsByIndex(repoID, index int64) (*Issue, error) {
 	issue, err := GetIssueByIndex(repoID, index)

models/issue_test.go

@@ -8,11 +8,13 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/foreignreference"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -534,3 +536,35 @@ func TestCorrectIssueStats(t *testing.T) {
 	assert.NoError(t, err)
 	assert.EqualValues(t, issueStats.OpenCount, issueAmount)
 }
+
+func TestIssueForeignReference(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 4}).(*Issue)
+	assert.NotEqualValues(t, issue.Index, issue.ID) // make sure they are different to avoid false positive
+
+	// it is fine for an issue to not have a foreign reference
+	err := issue.LoadAttributes()
+	assert.NoError(t, err)
+	assert.Nil(t, issue.ForeignReference)
+
+	var foreignIndex int64 = 12345
+	_, err = GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex)
+	assert.True(t, foreignreference.IsErrLocalIndexNotExist(err))
+
+	_, err = db.GetEngine(db.DefaultContext).Insert(&foreignreference.ForeignReference{
+		LocalIndex:   issue.Index,
+		ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+		RepoID:       issue.RepoID,
+		Type:         foreignreference.TypeIssue,
+	})
+	assert.NoError(t, err)
+
+	err = issue.LoadAttributes()
+	assert.NoError(t, err)
+
+	assert.EqualValues(t, issue.ForeignReference.ForeignIndex, strconv.FormatInt(foreignIndex, 10))
+
+	found, err := GetIssueByForeignIndex(context.Background(), issue.RepoID, foreignIndex)
+	assert.NoError(t, err)
+	assert.EqualValues(t, found.Index, issue.Index)
+}

models/migrate.go

@@ -83,6 +83,13 @@ func insertIssue(ctx context.Context, issue *Issue) error {
 		}
 	}
 
+	if issue.ForeignReference != nil {
+		issue.ForeignReference.LocalIndex = issue.Index
+		if _, err := sess.Insert(issue.ForeignReference); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

models/migrate_test.go

@@ -5,8 +5,10 @@
 package models
 
 import (
+	"strconv"
 	"testing"
 
+	"code.gitea.io/gitea/models/foreignreference"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -45,6 +47,7 @@ func assertCreateIssues(t *testing.T, isPull bool) {
 		UserID: owner.ID,
 	}
 
+	foreignIndex := int64(12345)
 	title := "issuetitle1"
 	is := &Issue{
 		RepoID:      repo.ID,
@@ -58,11 +61,20 @@ func assertCreateIssues(t *testing.T, isPull bool) {
 		IsClosed:    true,
 		Labels:      []*Label{label},
 		Reactions:   []*Reaction{reaction},
+		ForeignReference: &foreignreference.ForeignReference{
+			ForeignIndex: strconv.FormatInt(foreignIndex, 10),
+			RepoID:       repo.ID,
+			Type:         foreignreference.TypeIssue,
+		},
 	}
 	err := InsertIssues(is)
 	assert.NoError(t, err)
 
 	i := unittest.AssertExistsAndLoadBean(t, &Issue{Title: title}).(*Issue)
+	assert.Nil(t, i.ForeignReference)
+	err = i.LoadAttributes()
+	assert.NoError(t, err)
+	assert.EqualValues(t, strconv.FormatInt(foreignIndex, 10), i.ForeignReference.ForeignIndex)
 	unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID})
 }
 

models/migrations/migrations.go

@@ -373,6 +373,8 @@ var migrations = []Migration{
 	NewMigration("Increase WebAuthentication CredentialID size to 410 - NO-OPED", increaseCredentialIDTo410),
 	// v210 -> v211
 	NewMigration("v208 was completely broken - remigrate", remigrateU2FCredentials),
+	// v211 -> v212
+	NewMigration("Create ForeignReference table", createForeignReferenceTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v211.go

@@ -0,0 +1,26 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+func createForeignReferenceTable(x *xorm.Engine) error {
+	type ForeignReference struct {
+		// RepoID is the first column in all indices. now we only need 2 indices: (repo, local) and (repo, foreign, type)
+		RepoID       int64  `xorm:"UNIQUE(repo_foreign_type) INDEX(repo_local)" `
+		LocalIndex   int64  `xorm:"INDEX(repo_local)"` // the resource key inside Gitea, it can be IssueIndex, or some model ID.
+		ForeignIndex string `xorm:"INDEX UNIQUE(repo_foreign_type)"`
+		Type         string `xorm:"VARCHAR(16) INDEX UNIQUE(repo_foreign_type)"`
+	}
+
+	if err := x.Sync2(new(ForeignReference)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

modules/migration/comment.go

@@ -7,6 +7,13 @@ package migration
 
 import "time"
 
+// Commentable can be commented upon
+type Commentable interface {
+	GetLocalIndex() int64
+	GetForeignIndex() int64
+	GetContext() DownloaderContext
+}
+
 // Comment is a standard comment information
 type Comment struct {
 	IssueIndex  int64 `yaml:"issue_index"`

modules/migration/downloader.go

@@ -11,13 +11,6 @@ import (
 	"code.gitea.io/gitea/modules/structs"
 )
 
-// GetCommentOptions represents an options for get comment
-type GetCommentOptions struct {
-	Context  IssueContext
-	Page     int
-	PageSize int
-}
-
 // Downloader downloads the site repo information
 type Downloader interface {
 	SetContext(context.Context)
@@ -27,10 +20,11 @@ type Downloader interface {
 	GetReleases() ([]*Release, error)
 	GetLabels() ([]*Label, error)
 	GetIssues(page, perPage int) ([]*Issue, bool, error)
-	GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
+	GetComments(commentable Commentable) ([]*Comment, bool, error)
+	GetAllComments(page, perPage int) ([]*Comment, bool, error)
 	SupportGetRepoComments() bool
 	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
-	GetReviews(pullRequestContext IssueContext) ([]*Review, error)
+	GetReviews(reviewable Reviewable) ([]*Review, error)
 	FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
 }
 
@@ -39,3 +33,6 @@ type DownloaderFactory interface {
 	New(ctx context.Context, opts MigrateOptions) (Downloader, error)
 	GitServiceType() structs.GitServiceType
 }
+
+// DownloaderContext has opaque information only relevant to a given downloader
+type DownloaderContext interface{}