Skip to content

Commit 151daf7

Browse files
lunnylunny
authored
Fix bug isEnd detection on getIssues/getPullRequests (#13299) (#13301)
1 parent e177728 commit 151daf7

File tree

2 files changed

+50
-32
lines changed

2 files changed

+50
-32
lines changed

‎modules/migrations/github.go‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -65,23 +65,25 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
6565
// GithubDownloaderV3 implements a Downloader interface to get repository informations
6666
// from github via APIv3
6767
type GithubDownloaderV3 struct {
68-
ctx context.Context
69-
client *github.Client
70-
repoOwner string
71-
repoName string
72-
userName string
73-
password string
74-
rate *github.Rate
68+
ctx context.Context
69+
client *github.Client
70+
repoOwner string
71+
repoName string
72+
userName string
73+
password string
74+
rate *github.Rate
75+
maxPerPage int
7576
}
7677

7778
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
7879
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
7980
var downloader = GithubDownloaderV3{
80-
userName: userName,
81-
password: password,
82-
ctx: ctx,
83-
repoOwner: repoOwner,
84-
repoName: repoName,
81+
userName: userName,
82+
password: password,
83+
ctx: ctx,
84+
repoOwner: repoOwner,
85+
repoName: repoName,
86+
maxPerPage: 100,
8587
}
8688

8789
client := &http.Client{
@@ -177,7 +179,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
177179

178180
// GetMilestones returns milestones
179181
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
180-
var perPage = 100
182+
var perPage = g.maxPerPage
181183
var milestones = make([]*base.Milestone, 0, perPage)
182184
for i := 1; ; i++ {
183185
g.sleep()
@@ -233,7 +235,7 @@ func convertGithubLabel(label *github.Label) *base.Label {
233235

234236
// GetLabels returns labels
235237
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
236-
var perPage = 100
238+
var perPage = g.maxPerPage
237239
var labels = make([]*base.Label, 0, perPage)
238240
for i := 1; ; i++ {
239241
g.sleep()
@@ -304,7 +306,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
304306

305307
// GetReleases returns releases
306308
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
307-
var perPage = 100
309+
var perPage = g.maxPerPage
308310
var releases = make([]*base.Release, 0, perPage)
309311
for i := 1; ; i++ {
310312
g.sleep()
@@ -342,6 +344,9 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err
342344

343345
// GetIssues returns issues according start and limit
344346
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
347+
if perPage > g.maxPerPage {
348+
perPage = g.maxPerPage
349+
}
345350
opt := &github.IssueListByRepoOptions{
346351
Sort: "created",
347352
Direction: "asc",
@@ -429,15 +434,15 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
429434
// GetComments returns comments according issueNumber
430435
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
431436
var (
432-
allComments = make([]*base.Comment, 0, 100)
437+
allComments = make([]*base.Comment, 0, g.maxPerPage)
433438
created = "created"
434439
asc = "asc"
435440
)
436441
opt := &github.IssueListCommentsOptions{
437442
Sort: &created,
438443
Direction: &asc,
439444
ListOptions: github.ListOptions{
440-
PerPage: 100,
445+
PerPage: g.maxPerPage,
441446
},
442447
}
443448
for {
@@ -459,7 +464,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
459464
g.sleep()
460465
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
461466
Page: i,
462-
PerPage: 100,
467+
PerPage: g.maxPerPage,
463468
})
464469
if err != nil {
465470
return nil, err
@@ -497,6 +502,9 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
497502

498503
// GetPullRequests returns pull requests according page and perPage
499504
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
505+
if perPage > g.maxPerPage {
506+
perPage = g.maxPerPage
507+
}
500508
opt := &github.PullRequestListOptions{
501509
Sort: "created",
502510
Direction: "asc",
@@ -650,7 +658,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
650658
g.sleep()
651659
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
652660
Page: i,
653-
PerPage: 100,
661+
PerPage: g.maxPerPage,
654662
})
655663
if err != nil {
656664
return nil, err
@@ -687,9 +695,9 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
687695

688696
// GetReviews returns pull requests review
689697
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
690-
var allReviews = make([]*base.Review, 0, 100)
698+
var allReviews = make([]*base.Review, 0, g.maxPerPage)
691699
opt := &github.ListOptions{
692-
PerPage: 100,
700+
PerPage: g.maxPerPage,
693701
}
694702
for {
695703
g.sleep()
@@ -703,7 +711,7 @@ func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review
703711
r.IssueIndex = pullRequestNumber
704712
// retrieve all review comments
705713
opt2 := &github.ListOptions{
706-
PerPage: 100,
714+
PerPage: g.maxPerPage,
707715
}
708716
for {
709717
g.sleep()

‎modules/migrations/gitlab.go‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ type GitlabDownloader struct {
6868
repoName string
6969
issueCount int64
7070
fetchPRcomments bool
71+
maxPerPage int
7172
}
7273

7374
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
@@ -99,10 +100,11 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
99100
}
100101

101102
return &GitlabDownloader{
102-
ctx: ctx,
103-
client: gitlabClient,
104-
repoID: gr.ID,
105-
repoName: gr.Name,
103+
ctx: ctx,
104+
client: gitlabClient,
105+
repoID: gr.ID,
106+
repoName: gr.Name,
107+
maxPerPage: 100,
106108
}, nil
107109
}
108110

@@ -159,7 +161,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
159161

160162
// GetMilestones returns milestones
161163
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
162-
var perPage = 100
164+
var perPage = g.maxPerPage
163165
var state = "all"
164166
var milestones = make([]*base.Milestone, 0, perPage)
165167
for i := 1; ; i++ {
@@ -230,7 +232,7 @@ func (g *GitlabDownloader) normalizeColor(val string) string {
230232

231233
// GetLabels returns labels
232234
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
233-
var perPage = 100
235+
var perPage = g.maxPerPage
234236
var labels = make([]*base.Label, 0, perPage)
235237
for i := 1; ; i++ {
236238
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
@@ -281,7 +283,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
281283

282284
// GetReleases returns releases
283285
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
284-
var perPage = 100
286+
var perPage = g.maxPerPage
285287
var releases = make([]*base.Release, 0, perPage)
286288
for i := 1; ; i++ {
287289
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
@@ -330,6 +332,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
330332
state := "all"
331333
sort := "asc"
332334

335+
if perPage > g.maxPerPage {
336+
perPage = g.maxPerPage
337+
}
338+
333339
opt := &gitlab.ListProjectIssuesOptions{
334340
State: &state,
335341
Sort: &sort,
@@ -401,7 +407,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
401407
// GetComments returns comments according issueNumber
402408
// TODO: figure out how to transfer comment reactions
403409
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
404-
var allComments = make([]*base.Comment, 0, 100)
410+
var allComments = make([]*base.Comment, 0, g.maxPerPage)
405411

406412
var page = 1
407413
var realIssueNumber int64
@@ -415,14 +421,14 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
415421
realIssueNumber = issueNumber
416422
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
417423
Page: page,
418-
PerPage: 100,
424+
PerPage: g.maxPerPage,
419425
}, nil, gitlab.WithContext(g.ctx))
420426
} else {
421427
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
422428
realIssueNumber = issueNumber - g.issueCount
423429
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
424430
Page: page,
425-
PerPage: 100,
431+
PerPage: g.maxPerPage,
426432
}, nil, gitlab.WithContext(g.ctx))
427433
}
428434

@@ -465,6 +471,10 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
465471

466472
// GetPullRequests returns pull requests according page and perPage
467473
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
474+
if perPage > g.maxPerPage {
475+
perPage = g.maxPerPage
476+
}
477+
468478
opt := &gitlab.ListProjectMergeRequestsOptions{
469479
ListOptions: gitlab.ListOptions{
470480
PerPage: perPage,

0 commit comments

Comments
 (0)