Detect migrating batch size (go-gitea#7353)

* Make migrating batch size as configurable

* detect different table batch insert size and remove config item

* remove unrelated changes

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/go-macaron/toolbox v0.0.0-20180818072302-a77f45a7ce90
 	github.com/go-redis/redis v6.15.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-xorm/builder v0.3.4 // indirect
 	github.com/go-xorm/core v0.6.0 // indirect
 	github.com/go-xorm/xorm v0.7.3-0.20190620151208-f1b4f8368459
 	github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561

go.sum

@@ -133,6 +133,8 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-xorm/builder v0.3.4 h1:FxkeGB4Cggdw3tPwutLCpfjng2jugfkg6LDMrd/KsoY=
+github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
 github.com/go-xorm/core v0.6.0 h1:tp6hX+ku4OD9khFZS8VGBDRY3kfVCtelPfmkgCyHxL0=
 github.com/go-xorm/core v0.6.0/go.mod h1:d8FJ9Br8OGyQl12MCclmYBuBqqxsyeedpXciV5Myih8=
 github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:9wScpmSP5A3Bk8V3XHWUcJmYTh+ZnlHVyc+A4oZYS3Y=

models/models.go

@@ -393,3 +393,9 @@ func FindByMaxID(maxID int64, limit int, results interface{}) error {
 		Limit(limit).
 		Find(results)
 }
+
+// MaxBatchInsertSize returns the table's max batch insert size
+func MaxBatchInsertSize(bean interface{}) int {
+	t := x.TableInfo(bean)
+	return 999 / len(t.ColumnsSeq())
+}

modules/migrations/base/uploader.go

@@ -7,6 +7,7 @@ package base
 
 // Uploader uploads all the informations of one repository
 type Uploader interface {
+	MaxBatchInsertSize(tp string) int
 	CreateRepo(repo *Repository, opts MigrateOptions) error
 	CreateMilestones(milestones ...*Milestone) error
 	CreateReleases(releases ...*Release) error

modules/migrations/gitea.go

@@ -53,6 +53,25 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea
 	}
 }
 
+// MaxBatchInsertSize returns the table's max batch insert size
+func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
+	switch tp {
+	case "issue":
+		return models.MaxBatchInsertSize(new(models.Issue))
+	case "comment":
+		return models.MaxBatchInsertSize(new(models.Comment))
+	case "milestone":
+		return models.MaxBatchInsertSize(new(models.Milestone))
+	case "label":
+		return models.MaxBatchInsertSize(new(models.Label))
+	case "release":
+		return models.MaxBatchInsertSize(new(models.Release))
+	case "pullrequest":
+		return models.MaxBatchInsertSize(new(models.PullRequest))
+	}
+	return 10
+}
+
 // CreateRepo creates a repository
 func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
 	owner, err := models.GetUserByName(g.repoOwner)

modules/migrations/migrate.go

@@ -91,8 +91,16 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 			return err
 		}
 
-		if err := uploader.CreateMilestones(milestones...); err != nil {
-			return err
+		msBatchSize := uploader.MaxBatchInsertSize("milestone")
+		for len(milestones) > 0 {
+			if len(milestones) < msBatchSize {
+				msBatchSize = len(milestones)
+			}
+
+			if err := uploader.CreateMilestones(milestones...); err != nil {
+				return err
+			}
+			milestones = milestones[msBatchSize:]
 		}
 	}
 
@@ -103,8 +111,16 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 			return err
 		}
 
-		if err := uploader.CreateLabels(labels...); err != nil {
-			return err
+		lbBatchSize := uploader.MaxBatchInsertSize("label")
+		for len(labels) > 0 {
+			if len(labels) < lbBatchSize {
+				lbBatchSize = len(labels)
+			}
+
+			if err := uploader.CreateLabels(labels...); err != nil {
+				return err
+			}
+			labels = labels[lbBatchSize:]
 		}
 	}
 
@@ -115,15 +131,27 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 			return err
 		}
 
-		if err := uploader.CreateReleases(releases...); err != nil {
-			return err
+		relBatchSize := uploader.MaxBatchInsertSize("release")
+		for len(releases) > 0 {
+			if len(releases) < relBatchSize {
+				relBatchSize = len(releases)
+			}
+
+			if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
+				return err
+			}
+			releases = releases[relBatchSize:]
 		}
 	}
 
+	var commentBatchSize = uploader.MaxBatchInsertSize("comment")
+
 	if opts.Issues {
 		log.Trace("migrating issues and comments")
+		var issueBatchSize = uploader.MaxBatchInsertSize("issue")
+
 		for i := 1; ; i++ {
-			issues, isEnd, err := downloader.GetIssues(i, 100)
+			issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
 			if err != nil {
 				return err
 			}
@@ -141,7 +169,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				continue
 			}
 
-			var allComments = make([]*base.Comment, 0, 100)
+			var allComments = make([]*base.Comment, 0, commentBatchSize)
 			for _, issue := range issues {
 				comments, err := downloader.GetComments(issue.Number)
 				if err != nil {
@@ -154,11 +182,12 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				}
 				allComments = append(allComments, comments...)
 
-				if len(allComments) >= 100 {
-					if err := uploader.CreateComments(allComments...); err != nil {
+				if len(allComments) >= commentBatchSize {
+					if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
 						return err
 					}
-					allComments = make([]*base.Comment, 0, 100)
+
+					allComments = allComments[commentBatchSize:]
 				}
 			}
 
@@ -176,8 +205,9 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.PullRequests {
 		log.Trace("migrating pull requests and comments")
+		var prBatchSize = models.MaxBatchInsertSize("pullrequest")
 		for i := 1; ; i++ {
-			prs, err := downloader.GetPullRequests(i, 100)
+			prs, err := downloader.GetPullRequests(i, prBatchSize)
 			if err != nil {
 				return err
 			}
@@ -195,7 +225,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				continue
 			}
 
-			var allComments = make([]*base.Comment, 0, 100)
+			var allComments = make([]*base.Comment, 0, commentBatchSize)
 			for _, pr := range prs {
 				comments, err := downloader.GetComments(pr.Number)
 				if err != nil {
@@ -209,11 +239,11 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 				allComments = append(allComments, comments...)
 
-				if len(allComments) >= 100 {
-					if err := uploader.CreateComments(allComments...); err != nil {
+				if len(allComments) >= commentBatchSize {
+					if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
 						return err
 					}
-					allComments = make([]*base.Comment, 0, 100)
+					allComments = allComments[commentBatchSize:]
 				}
 			}
 			if len(allComments) > 0 {
@@ -222,7 +252,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				}
 			}
 
-			if len(prs) < 100 {
+			if len(prs) < prBatchSize {
 				break
 			}
 		}