Skip to content

Commit dbc9caf

Browse files
carlosmscarlosms
committed
Add a new shallow cmd that uses list endpints, no queue
Signed-off-by: Carlos Martín <carlos.martin.sanchez@gmail.com>
1 parent 190f44a commit dbc9caf

File tree

8 files changed

+486
-0
lines changed

8 files changed

+486
-0
lines changed

‎cmd/ghsync/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ var (
1717
var app = cli.New("ghsync", version, build, "GitHub metadata sync")
1818

1919
func main() {
20+
app.AddCommand(&subcmd.ShallowCommand{})
2021
app.AddCommand(&subcmd.SyncCommand{})
2122
app.AddCommand(&subcmd.MigrateCommand{})
2223

‎cmd/ghsync/subcmd/shallow.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package subcmd
2+
3+
import (
4+
"github.com/src-d/ghsync/shallow"
5+
6+
"gopkg.in/src-d/go-cli.v0"
7+
)
8+
9+
type ShallowCommand struct {
10+
cli.Command `name:"shallow" short-description:"Shallow sync of GitHub data" long-description:"Shallow sync of GitHub data"`
11+
12+
Token string `long:"token" env:"GHSYNC_TOKEN" description:"GitHub personal access token" required:"true"`
13+
Org string `long:"org" env:"GHSYNC_ORG" description:"Name of the GitHub organization" required:"true"`
14+
15+
Postgres PostgresOpt `group:"PostgreSQL connection options"`
16+
}
17+
18+
func (c *ShallowCommand) Execute(args []string) error {
19+
db, err := c.Postgres.initDB()
20+
if err != nil {
21+
return err
22+
}
23+
defer db.Close()
24+
25+
client, err := newClient(c.Token)
26+
if err != nil {
27+
return err
28+
}
29+
30+
orgSyncer := shallow.NewOrganizationSyncer(db, client)
31+
return orgSyncer.Sync(c.Org)
32+
}

‎shallow/common.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
package shallow
2+
3+
const listOptionsPerPage = 100

‎shallow/issue.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package shallow
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
7+
8+
"github.com/src-d/ghsync/models"
9+
10+
"github.com/google/go-github/github"
11+
"gopkg.in/src-d/go-kallax.v1"
12+
"gopkg.in/src-d/go-log.v1"
13+
)
14+
15+
type IssueSyncer struct {
16+
db *sql.DB
17+
store *models.IssueStore
18+
client *github.Client
19+
}
20+
21+
func NewIssueSyncer(db *sql.DB, c *github.Client) *IssueSyncer {
22+
return &IssueSyncer{
23+
db: db,
24+
store: models.NewIssueStore(db),
25+
client: c,
26+
}
27+
}
28+
29+
func (s *IssueSyncer) Sync(owner, repo string, logger log.Logger) error {
30+
opts := &github.IssueListByRepoOptions{}
31+
opts.ListOptions.PerPage = listOptionsPerPage
32+
opts.State = "all"
33+
34+
logger.Infof("starting to retrieve issues")
35+
36+
// TODO transaction for faster times
37+
38+
// Get the list of all issues
39+
for {
40+
issues, r, err := s.client.Issues.ListByRepo(context.TODO(), owner, repo, opts)
41+
if err != nil {
42+
return err
43+
}
44+
45+
for _, i := range issues {
46+
if i.IsPullRequest() {
47+
continue
48+
}
49+
50+
logger := logger.With(log.Fields{"issue": i.GetNumber()})
51+
52+
_, err := s.store.FindOne(models.NewIssueQuery().
53+
Where(kallax.And(
54+
kallax.Eq(models.Schema.Issue.RepositoryOwner, owner),
55+
kallax.Eq(models.Schema.Issue.RepositoryName, repo),
56+
kallax.Eq(models.Schema.Issue.Number, i.GetNumber()),
57+
)),
58+
)
59+
60+
if err != nil && err != kallax.ErrNotFound {
61+
logger.Errorf(err, "failed to read the resource from the DB")
62+
return fmt.Errorf("failed to read the resource from the DB: %v", err)
63+
}
64+
65+
if err == nil {
66+
logger.Infof("resource already exists, skipping")
67+
continue
68+
}
69+
70+
record := models.NewIssue()
71+
record.Issue = *i
72+
73+
err = s.store.Insert(record)
74+
if err != nil {
75+
logger.Errorf(err, "failed to write the resource into the DB")
76+
return fmt.Errorf("failed to write the resource into the DB: %v", err)
77+
}
78+
79+
logger.Debugf("resource written in the DB")
80+
}
81+
82+
if r.NextPage == 0 {
83+
break
84+
}
85+
86+
opts.Page = r.NextPage
87+
}
88+
89+
logger.Infof("finished to retrieve issues")
90+
91+
return nil
92+
}

‎shallow/organization.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package shallow
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
7+
8+
"github.com/src-d/ghsync/models"
9+
10+
"github.com/google/go-github/github"
11+
"gopkg.in/src-d/go-kallax.v1"
12+
"gopkg.in/src-d/go-log.v1"
13+
)
14+
15+
type OrganizationSyncer struct {
16+
db *sql.DB
17+
store *models.OrganizationStore
18+
client *github.Client
19+
}
20+
21+
func NewOrganizationSyncer(db *sql.DB, c *github.Client) *OrganizationSyncer {
22+
return &OrganizationSyncer{
23+
db: db,
24+
store: models.NewOrganizationStore(db),
25+
client: c,
26+
}
27+
}
28+
29+
func (s *OrganizationSyncer) Sync(login string) error {
30+
logger := log.With(log.Fields{"organization": login})
31+
32+
_, err := s.store.FindOne(models.NewOrganizationQuery().
33+
Where(kallax.Eq(models.Schema.Organization.Login, login)),
34+
)
35+
36+
if err != nil && err != kallax.ErrNotFound {
37+
logger.Errorf(err, "failed to read the resource from the DB")
38+
return fmt.Errorf("failed to read the resource from the DB: %v", err)
39+
}
40+
41+
if err == nil {
42+
logger.Infof("resource already exists, skipping")
43+
return nil
44+
}
45+
46+
org, _, err := s.client.Organizations.Get(context.TODO(), login)
47+
if err != nil {
48+
return err
49+
}
50+
51+
repoSyncer := NewRepositorySyncer(s.db, s.client)
52+
err = repoSyncer.Sync(login, logger)
53+
if err != nil {
54+
return err
55+
}
56+
57+
userSyncer := NewUserSyncer(s.db, s.client)
58+
err = userSyncer.Sync(login, logger)
59+
if err != nil {
60+
return err
61+
}
62+
63+
record := models.NewOrganization()
64+
record.Organization = *org
65+
66+
logger.Debugf("inserting resource")
67+
68+
err = s.store.Insert(record)
69+
if err != nil {
70+
logger.Errorf(err, "failed to write the resource into the DB")
71+
return fmt.Errorf("failed to write the resource into the DB: %v", err)
72+
}
73+
74+
logger.Debugf("resource written in the DB")
75+
76+
return nil
77+
}

‎shallow/pull_request.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package shallow
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
7+
8+
"github.com/src-d/ghsync/models"
9+
10+
"github.com/google/go-github/github"
11+
"gopkg.in/src-d/go-kallax.v1"
12+
"gopkg.in/src-d/go-log.v1"
13+
)
14+
15+
type PullRequestSyncer struct {
16+
db *sql.DB
17+
store *models.PullRequestStore
18+
client *github.Client
19+
}
20+
21+
func NewPullRequestSyncer(db *sql.DB, c *github.Client) *PullRequestSyncer {
22+
return &PullRequestSyncer{
23+
db: db,
24+
store: models.NewPullRequestStore(db),
25+
client: c,
26+
}
27+
}
28+
29+
func (s *PullRequestSyncer) Sync(owner, repo string, logger log.Logger) error {
30+
opts := &github.PullRequestListOptions{}
31+
opts.ListOptions.PerPage = listOptionsPerPage
32+
opts.State = "all"
33+
34+
logger.Infof("starting to retrieve PRs")
35+
36+
// TODO transaction for faster times
37+
38+
// Get the list of all PRs
39+
for {
40+
prs, r, err := s.client.PullRequests.List(context.TODO(), owner, repo, opts)
41+
if err != nil {
42+
return err
43+
}
44+
45+
for _, pr := range prs {
46+
logger := logger.With(log.Fields{"pr": pr.GetNumber()})
47+
48+
_, err := s.store.FindOne(models.NewPullRequestQuery().
49+
Where(kallax.And(
50+
kallax.Eq(models.Schema.Issue.RepositoryOwner, owner),
51+
kallax.Eq(models.Schema.Issue.RepositoryName, repo),
52+
kallax.Eq(models.Schema.Issue.Number, pr.GetNumber()),
53+
)),
54+
)
55+
56+
if err != nil && err != kallax.ErrNotFound {
57+
logger.Errorf(err, "failed to read the resource from the DB")
58+
return fmt.Errorf("failed to read the resource from the DB: %v", err)
59+
}
60+
61+
if err == nil {
62+
logger.Infof("resource already exists, skipping")
63+
continue
64+
}
65+
66+
record := models.NewPullRequest()
67+
record.PullRequest = *pr
68+
69+
err = s.store.Insert(record)
70+
if err != nil {
71+
logger.Errorf(err, "failed to write the resource into the DB")
72+
return fmt.Errorf("failed to write the resource into the DB: %v", err)
73+
}
74+
75+
logger.Debugf("resource written in the DB")
76+
}
77+
78+
if r.NextPage == 0 {
79+
break
80+
}
81+
82+
opts.Page = r.NextPage
83+
}
84+
85+
logger.Infof("finished to retrieve PRs")
86+
87+
return nil
88+
}

0 commit comments

Comments
 (0)