Sync issues, PRs, labels, milestones, comments from GitHub mirrors

cmd/admin.go

@@ -135,7 +135,7 @@ func runRepoSyncReleases(_ *cli.Context) error {
 			}
 			log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
 
-			if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
+			if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, false); err != nil {
 				log.Warn(" SyncReleasesWithTags: %v", err)
 				gitRepo.Close()
 				continue

models/issues/comment.go

@@ -228,6 +228,7 @@ type Comment struct {
 	Poster           *user_model.User `xorm:"-"`
 	OriginalAuthor   string
 	OriginalAuthorID int64
+	OriginalID       int64  // Only used in synced comments
 	IssueID          int64  `xorm:"INDEX"`
 	Issue            *Issue `xorm:"-"`
 	LabelID          int64
@@ -1305,3 +1306,77 @@ func InsertIssueComments(ctx context.Context, comments []*Comment) error {
 	}
 	return committer.Commit()
 }
+
+// UpsertIssueComments inserts many comments of issues.
+func UpsertIssueComments(ctx context.Context, comments []*Comment) error {
+	if len(comments) == 0 {
+		return nil
+	}
+
+	issueIDs := make(map[int64]bool)
+	for _, comment := range comments {
+		issueIDs[comment.IssueID] = true
+	}
+
+	return db.WithTx(ctx, func(ctx context.Context) error {
+		sess := db.GetEngine(ctx)
+		for _, comment := range comments {
+			exists, err := sess.Exist(&Comment{
+				IssueID:    comment.IssueID,
+				OriginalID: comment.OriginalID,
+			})
+			if err != nil {
+				return err
+			}
+			if !exists {
+				if _, err := sess.NoAutoTime().Insert(comment); err != nil {
+					return err
+				}
+			} else {
+				if _, err := sess.NoAutoTime().Where(
+					"issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID,
+				).AllCols().Update(comment); err != nil {
+					return err
+				}
+			}
+
+			for _, reaction := range comment.Reactions {
+				reaction.IssueID = comment.IssueID
+				reaction.CommentID = comment.ID
+			}
+			if len(comment.Reactions) > 0 {
+				for _, reaction := range comment.Reactions {
+					// issue comment reaction is uniquely identified by issue_id, comment_id and type
+					exists, err := sess.Exist(&Reaction{
+						IssueID:   reaction.IssueID,
+						CommentID: reaction.CommentID,
+						Type:      reaction.Type,
+					})
+					if err != nil {
+						return err
+					}
+					if exists {
+						if _, err := sess.Where(
+							"issue_id = ? AND comment_id = ? AND type = ?",
+							reaction.IssueID, reaction.CommentID, reaction.Type,
+						).AllCols().Update(&reaction); err != nil {
+							return err
+						}
+					} else {
+						if _, err := sess.Insert(&reaction); err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+
+		for issueID := range issueIDs {
+			if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?",
+				issueID, CommentTypeComment, issueID); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

models/issues/issue.go

@@ -909,3 +909,81 @@ func insertIssue(ctx context.Context, issue *Issue) error {
 
 	return nil
 }
+
+// UpsertIssues creates new issues and updates existing issues in database
+func UpsertIssues(ctx context.Context, issues ...*Issue) error {
+	return db.WithTx(ctx, func(ctx context.Context) error {
+		for _, issue := range issues {
+			if _, err := upsertIssue(ctx, issue); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func upsertIssue(ctx context.Context, issue *Issue) (isInsert bool, err error) {
+	sess := db.GetEngine(ctx)
+	has, err := sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Get(&issue.ID)
+	if err != nil {
+		return false, err
+	}
+	if !has {
+		return true, insertIssue(ctx, issue)
+	}
+	return false, updateIssue(ctx, issue)
+}
+
+func updateIssue(ctx context.Context, issue *Issue) error {
+	sess := db.GetEngine(ctx)
+	if _, err := sess.NoAutoTime().ID(issue.ID).AllCols().Update(issue); err != nil {
+		return err
+	}
+	issueLabels := resolveIssueLabels(issue.ID, issue.Labels)
+	if len(issueLabels) > 0 {
+		// delete old labels
+		if _, err := sess.Table("issue_label").Where("issue_id = ?", issue.ID).Delete(); err != nil {
+			return err
+		}
+		// insert new labels
+		if _, err := sess.Insert(issueLabels); err != nil {
+			return err
+		}
+	}
+
+	for _, reaction := range issue.Reactions {
+		reaction.IssueID = issue.ID
+	}
+
+	if len(issue.Reactions) > 0 {
+		// update existing reactions and insert new ones
+		for _, reaction := range issue.Reactions {
+			exists, err := sess.Exist(&Reaction{ID: reaction.ID})
+			if err != nil {
+				return err
+			}
+			if exists {
+				if _, err := sess.ID(reaction.ID).AllCols().Update(&reaction); err != nil {
+					return err
+				}
+			} else {
+				if _, err := sess.Insert(&reaction); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func resolveIssueLabels(issueID int64, labels []*Label) []IssueLabel {
+	issueLabels := make([]IssueLabel, 0, len(labels))
+	for _, label := range labels {
+		issueLabels = append(issueLabels, IssueLabel{
+			IssueID: issueID,
+			LabelID: label.ID,
+		})
+	}
+	return issueLabels
+}

models/issues/label.go

@@ -92,6 +92,8 @@ type Label struct {
 	CreatedUnix     timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix     timeutil.TimeStamp `xorm:"INDEX updated"`
 
+	OriginalID int64 // Only for migrating data from other system, used for syncing
+
 	NumOpenIssues     int    `xorm:"-"`
 	NumOpenRepoIssues int64  `xorm:"-"`
 	IsChecked         bool   `xorm:"-"`
@@ -390,6 +392,69 @@ func GetLabelsByRepoID(ctx context.Context, repoID int64, sortType string, listO
 	return labels, sess.Find(&labels)
 }
 
+// UpdateLabelsByRepoID adds, updates, and deletes relevant labels for the given repository.
+func UpdateLabelsByRepoID(ctx context.Context, repoID int64, labels ...*Label) error {
+	return db.WithTx(ctx, func(ctx context.Context) error {
+		existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{})
+		if err != nil {
+			return err
+		}
+		labelsToAdd := make([]*Label, 0)
+		labelsToUpdate := make([]*Label, 0)
+		labelsToDelete := make([]*Label, 0)
+
+		for _, l := range labels {
+			var foundLabel *Label
+			for _, existingLabel := range existingLabels {
+				if existingLabel.OriginalID == l.OriginalID {
+					foundLabel = existingLabel
+					break
+				}
+			}
+
+			if foundLabel == nil {
+				labelsToAdd = append(labelsToAdd, l)
+			} else if foundLabel.Name != l.Name || foundLabel.Description != l.Description ||
+				foundLabel.Color != l.Color {
+				l.RepoID = repoID
+				labelsToUpdate = append(labelsToUpdate, l)
+			}
+		}
+
+		for _, existingLabel := range existingLabels {
+			found := false
+			for _, label := range labels {
+				if label.OriginalID == existingLabel.OriginalID {
+					found = true
+					break
+				}
+			}
+			if !found {
+				labelsToDelete = append(labelsToDelete, existingLabel)
+			}
+		}
+
+		for _, l := range labelsToAdd {
+			if err = NewLabel(ctx, l); err != nil {
+				return err
+			}
+		}
+
+		for _, l := range labelsToUpdate {
+			if err = UpdateLabel(ctx, l); err != nil {
+				return err
+			}
+		}
+
+		for _, l := range labelsToDelete {
+			if err = DeleteLabel(ctx, repoID, l.ID); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
 // CountLabelsByRepoID count number of all labels that belong to given repository by ID.
 func CountLabelsByRepoID(ctx context.Context, repoID int64) (int64, error) {
 	return db.GetEngine(ctx).Where("repo_id = ?", repoID).Count(&Label{})

models/issues/milestone.go

@@ -65,6 +65,8 @@ type Milestone struct {
 	DeadlineString string `xorm:"-"`
 
 	TotalTrackedTime int64 `xorm:"-"`
+
+	OriginalID int64 // ID from the upstream syncing source
 }
 
 func init() {
@@ -380,3 +382,85 @@ func InsertMilestones(ctx context.Context, ms ...*Milestone) (err error) {
 	}
 	return committer.Commit()
 }
+
+// UpdateMilestones updates milestones of repository.
+func UpdateMilestones(ctx context.Context, ms ...*Milestone) (err error) {
+	if len(ms) == 0 {
+		return nil
+	}
+
+	return db.WithTx(ctx, func(ctx context.Context) error {
+		sess := db.GetEngine(ctx)
+
+		// get existing milestones
+		existingMilestones := make([]*Milestone, 0)
+		if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil {
+			return err
+		}
+
+		milestonesToAdd := make([]*Milestone, 0)
+		milestonesToUpdate := make([]*Milestone, 0)
+		milestonesToDelete := make([]*Milestone, 0)
+		foundMap := make(map[int64]bool)
+
+		openCount := 0
+		closedCount := 0
+
+		for _, m := range ms {
+			var foundMilestone *Milestone
+			for _, existingMilestone := range existingMilestones {
+				if existingMilestone.OriginalID == m.OriginalID {
+					foundMilestone = existingMilestone
+					foundMap[existingMilestone.ID] = true
+					break
+				}
+			}
+
+			if foundMilestone == nil {
+				milestonesToAdd = append(milestonesToAdd, m)
+			} else if foundMilestone.OriginalID != m.OriginalID {
+				m.ID = foundMilestone.ID
+				milestonesToUpdate = append(milestonesToUpdate, m)
+			}
+
+			if m.IsClosed {
+				closedCount++
+			} else {
+				openCount++
+			}
+		}
+
+		for _, existingMilestone := range existingMilestones {
+			if _, exist := foundMap[existingMilestone.ID]; !exist {
+				milestonesToDelete = append(milestonesToDelete, existingMilestone)
+			}
+		}
+
+		if len(milestonesToAdd) > 0 {
+			if _, err = sess.Insert(milestonesToAdd); err != nil {
+				return err
+			}
+		}
+
+		for _, m := range milestonesToUpdate {
+			if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil {
+				return err
+			}
+		}
+
+		for _, m := range milestonesToDelete {
+			if _, err = sess.ID(m.ID).Delete(m); err != nil {
+				return err
+			}
+		}
+
+		if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{
+			NumMilestones:       len(ms),
+			NumOpenMilestones:   openCount,
+			NumClosedMilestones: closedCount,
+		}); err != nil {
+			return err
+		}
+		return nil
+	})
+}

models/issues/pull.go

@@ -1046,6 +1046,35 @@ func InsertPullRequests(ctx context.Context, prs ...*PullRequest) error {
 	return committer.Commit()
 }
 
+// UpsertPullRequests inserts new pull requests and updates existing pull requests in database
+func UpsertPullRequests(ctx context.Context, prs ...*PullRequest) error {
+	if len(prs) == 0 {
+		return nil
+	}
+
+	return db.WithTx(ctx, func(ctx context.Context) error {
+		sess := db.GetEngine(ctx)
+		for _, pr := range prs {
+			isInsert, err := upsertIssue(ctx, pr.Issue)
+			if err != nil {
+				return err
+			}
+			pr.IssueID = pr.Issue.ID
+
+			if isInsert {
+				if _, err := sess.NoAutoTime().Insert(pr); err != nil {
+					return err
+				}
+			} else {
+				if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+}
+
 // GetPullRequestByMergedCommit returns a merged pull request by the given commit
 func GetPullRequestByMergedCommit(ctx context.Context, repoID int64, sha string) (*PullRequest, error) {
 	pr := new(PullRequest)