Skip to content

Commit a386f21

Browse files
carlosmscarlosms
authored
Merge pull request #326 from carlosms/git-auth
Use github auth to fetch git contents
2 parents 4a21a57 + ac2f4ed commit a386f21

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+3513
-1161
lines changed

‎Gopkg.lock

+6-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/lookoutd/common.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,17 @@ func (c *queueConsumerCommand) initDataHandler() (*lookout.DataServerHandler, er
326326
return nil, err
327327
}
328328

329+
var authProvider git.AuthProvider
330+
if c.Provider == github.Provider {
331+
if c.pool == nil {
332+
return nil, fmt.Errorf("pool must be initialized with initProvider")
333+
}
334+
335+
authProvider = c.pool
336+
}
337+
329338
lib := git.NewLibrary(osfs.New(c.Library))
330-
sync := git.NewSyncer(lib)
339+
sync := git.NewSyncer(lib, authProvider)
331340
loader := git.NewLibraryCommitLoader(lib, sync)
332341

333342
gitService := git.NewService(loader)

‎cmd/lookoutd/serve.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ func (c *ServeCommand) Execute(args []string) error {
2828
return err
2929
}
3030

31+
err = c.initProvider(conf)
32+
if err != nil {
33+
return err
34+
}
35+
3136
dataHandler, err := c.initDataHandler()
3237
if err != nil {
3338
return err
@@ -49,11 +54,6 @@ func (c *ServeCommand) Execute(args []string) error {
4954
return err
5055
}
5156

52-
err = c.initProvider(conf)
53-
if err != nil {
54-
return err
55-
}
56-
5757
poster, err := c.initPoster(conf)
5858
if err != nil {
5959
return err

‎cmd/lookoutd/work.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ func (c *WorkCommand) Execute(args []string) error {
2828
return err
2929
}
3030

31+
err = c.initProvider(conf)
32+
if err != nil {
33+
return err
34+
}
35+
3136
dataHandler, err := c.initDataHandler()
3237
if err != nil {
3338
return err
@@ -49,11 +54,6 @@ func (c *WorkCommand) Execute(args []string) error {
4954
return err
5055
}
5156

52-
err = c.initProvider(conf)
53-
if err != nil {
54-
return err
55-
}
56-
5757
poster, err := c.initPoster(conf)
5858
if err != nil {
5959
return err

‎dummy/dummy_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/stretchr/testify/suite"
1313
"google.golang.org/grpc"
1414
"gopkg.in/src-d/go-git-fixtures.v3"
15+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1516
"gopkg.in/src-d/go-git.v4/storage/filesystem"
1617
log "gopkg.in/src-d/go-log.v1"
1718
)
@@ -42,8 +43,7 @@ func (s *DummySuite) SetupSuite() {
4243
fixture := fixtures.Basic().One()
4344
s.Basic = fixture
4445
fs := fixture.DotGit()
45-
sto, err := filesystem.NewStorage(fs)
46-
require.NoError(err)
46+
sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize))
4747

4848
s.apiServer = grpc.NewServer()
4949
server := &lookout.DataServerHandler{

‎provider/github/client.go

+25-1
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
11
package github
22

33
import (
4+
"context"
45
"net/http"
56
"strconv"
67
"strings"
78
"sync"
89
"time"
910

1011
"github.com/src-d/lookout"
12+
"github.com/src-d/lookout/service/git"
1113
"github.com/src-d/lookout/util/cache"
1214
"github.com/src-d/lookout/util/ctxlog"
1315

1416
"github.com/google/go-github/github"
1517
"github.com/gregjones/httpcache"
18+
"gopkg.in/src-d/go-git.v4/plumbing/transport"
1619
log "gopkg.in/src-d/go-log.v1"
1720
)
1821

@@ -214,16 +217,36 @@ func (p *ClientPool) notify(e ClientPoolEvent) {
214217
}
215218
}
216219

220+
var _ git.AuthProvider = &ClientPool{}
221+
222+
// GitAuth returns a go-git auth method for a repo
223+
func (p *ClientPool) GitAuth(ctx context.Context, repoInfo *lookout.RepositoryInfo) transport.AuthMethod {
224+
c, ok := p.Client(repoInfo.Username, repoInfo.Name)
225+
if !ok {
226+
return nil
227+
}
228+
229+
return c.gitAuth(ctx)
230+
}
231+
232+
type gitAuthFn = func(ctx context.Context) transport.AuthMethod
233+
217234
// Client is a wrapper for github.Client that supports cache and provides rate limit information
218235
type Client struct {
219236
*github.Client
220237
cache *cache.ValidableCache
221238
limitRT *limitRoundTripper
222239
watchMinInterval time.Duration
240+
gitAuth gitAuthFn
223241
}
224242

225243
// NewClient creates new Client
226-
func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterval string) *Client {
244+
func NewClient(
245+
t http.RoundTripper,
246+
cache *cache.ValidableCache,
247+
watchMinInterval string,
248+
gitAuth gitAuthFn,
249+
) *Client {
227250
limitRT := &limitRoundTripper{
228251
Base: t,
229252
}
@@ -247,6 +270,7 @@ func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterva
247270
cache: cache,
248271
limitRT: limitRT,
249272
watchMinInterval: interval,
273+
gitAuth: gitAuth,
250274
}
251275
}
252276

‎provider/github/installations.go

+19-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ import (
88
"github.com/google/go-github/github"
99
"github.com/src-d/lookout"
1010
"github.com/src-d/lookout/util/cache"
11+
"github.com/src-d/lookout/util/ctxlog"
1112
vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1"
13+
"gopkg.in/src-d/go-git.v4/plumbing/transport"
14+
githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
1215
log "gopkg.in/src-d/go-log.v1"
1316
)
1417

@@ -123,9 +126,24 @@ func (t *Installations) createClient(installationID int64) (*Client, error) {
123126
return nil, err
124127
}
125128

129+
// Auth must be: https://x-access-token:<token>@github.com/owner/repo.git
130+
// Reference: https://developer.github.com/apps/building-github-apps/authenticating-with-github-apps/#http-based-git-access-by-an-installation
131+
gitAuth := func(ctx context.Context) transport.AuthMethod {
132+
token, err := itr.Token()
133+
if err != nil {
134+
ctxlog.Get(ctx).Errorf(err, "failed to get an installation access token")
135+
return nil
136+
}
137+
138+
return &githttp.BasicAuth{
139+
Username: "x-access-token",
140+
Password: token,
141+
}
142+
}
143+
126144
// TODO (carlosms): hardcoded, take from config
127145
watchMinInterval := ""
128-
return NewClient(itr, t.cache, watchMinInterval), nil
146+
return NewClient(itr, t.cache, watchMinInterval, gitAuth), nil
129147
}
130148

131149
func (t *Installations) getRepos(iClient *Client) ([]*lookout.RepositoryInfo, error) {

‎provider/github/poster.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package github
33
import (
44
"context"
55
"fmt"
6+
"reflect"
67
"strings"
78

89
"github.com/src-d/lookout"
@@ -66,7 +67,7 @@ func (p *Poster) Post(ctx context.Context, e lookout.Event,
6667

6768
return p.postPR(ctx, ev, aCommentsList)
6869
default:
69-
return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type"))
70+
return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e)))
7071
}
7172
}
7273

@@ -302,7 +303,7 @@ func (p *Poster) Status(ctx context.Context, e lookout.Event, status lookout.Ana
302303

303304
return p.statusPR(ctx, ev, status)
304305
default:
305-
return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type"))
306+
return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e)))
306307
}
307308
}
308309

‎provider/github/token_transport.go

+16-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
package github
22

33
import (
4+
"context"
45
"net/http"
56

67
"github.com/src-d/lookout"
78
"github.com/src-d/lookout/util/cache"
89
"github.com/src-d/lookout/util/ctxlog"
910

1011
vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1"
12+
"gopkg.in/src-d/go-git.v4/plumbing/transport"
13+
githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
1114
log "gopkg.in/src-d/go-log.v1"
1215
)
1316

@@ -42,10 +45,21 @@ func NewClientPoolFromTokens(urlToConfig map[string]ClientConfig, cache *cache.V
4245
byClients := make(map[*Client][]*lookout.RepositoryInfo, len(byConfig))
4346
byRepo := make(map[string]*Client, len(urlToConfig))
4447
for conf, repos := range byConfig {
45-
client := NewClient(&roundTripper{
48+
rt := &roundTripper{
4649
User: conf.User,
4750
Password: conf.Token,
48-
}, cache, conf.MinInterval)
51+
}
52+
53+
// Auth must be: https://<token>@github.com/owner/repo.git
54+
// Reference: https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/
55+
gitAuth := func(ctx context.Context) transport.AuthMethod {
56+
return &githttp.BasicAuth{
57+
Username: conf.Token,
58+
Password: "",
59+
}
60+
}
61+
62+
client := NewClient(rt, cache, conf.MinInterval, gitAuth)
4963

5064
if _, ok := byClients[client]; !ok {
5165
byClients[client] = []*lookout.RepositoryInfo{}

‎provider/github/utils.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/google/go-github/github"
1212
"gopkg.in/sourcegraph/go-vcsurl.v1"
1313
"gopkg.in/src-d/go-git.v4/plumbing"
14+
log "gopkg.in/src-d/go-log.v1"
1415
)
1516

1617
func castEvent(r *lookout.RepositoryInfo, e *github.Event) (lookout.Event, error) {
@@ -100,7 +101,10 @@ func castPullRequestBranch(ctx context.Context, b *github.PullRequestBranch) loo
100101

101102
r, err := vcsurl.Parse(b.GetRepo().GetCloneURL())
102103
if err != nil {
103-
ctxlog.Get(ctx).Warningf("malformed repository URL on pull request branch")
104+
ctxlog.Get(ctx).With(log.Fields{
105+
"url": b.GetRepo().GetCloneURL()},
106+
).Warningf("malformed repository URL on pull request branch")
107+
104108
return lookout.ReferencePointer{}
105109
}
106110

‎provider/github/watcher_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ func (s *WatcherTestSuite) TestCustomMinInterval() {
296296
s.mux.HandleFunc("/repos/mock/test/events", eventsHandler(&eventCalls))
297297

298298
clientMinInterval := 200 * time.Millisecond
299-
client := NewClient(nil, s.cache, clientMinInterval.String())
299+
client := NewClient(nil, s.cache, clientMinInterval.String(), nil)
300300
client.BaseURL = s.githubURL
301301
client.UploadURL = s.githubURL
302302

@@ -483,7 +483,7 @@ func (t *NoopTransport) Get(repo string) http.RoundTripper {
483483
}
484484

485485
func newClient(githubURL *url.URL, cache *cache.ValidableCache) *Client {
486-
client := NewClient(nil, cache, "")
486+
client := NewClient(nil, cache, "", nil)
487487
client.BaseURL = githubURL
488488
client.UploadURL = githubURL
489489
return client

‎server/server.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package server
33
import (
44
"context"
55
"fmt"
6+
"reflect"
67
"sync"
78

89
"github.com/src-d/lookout"
@@ -40,7 +41,7 @@ func NewServer(p lookout.Poster, fileGetter lookout.FileGetter,
4041
// HandleEvent processes the event calling the analyzers, and posting the results
4142
func (s *Server) HandleEvent(ctx context.Context, e lookout.Event) error {
4243
ctx, logger := ctxlog.WithLogFields(ctx, log.Fields{
43-
"event-type": e.Type(),
44+
"event-type": reflect.TypeOf(e),
4445
"event-id": e.ID().String(),
4546
"repo": e.Revision().Head.InternalRepositoryURL,
4647
"head": e.Revision().Head.ReferenceName,

‎service/git/examples_test.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/src-d/lookout"
88

99
"gopkg.in/src-d/go-git-fixtures.v3"
10+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1011
"gopkg.in/src-d/go-git.v4/storage/filesystem"
1112
)
1213

@@ -18,10 +19,7 @@ func Example() {
1819

1920
fixture := fixtures.Basic().One()
2021
fs := fixture.DotGit()
21-
storer, err := filesystem.NewStorage(fs)
22-
if err != nil {
23-
panic(err)
24-
}
22+
storer := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize))
2523

2624
// Create the git service with a repository loader that allows it to find
2725
// a repository by ID.

‎service/git/library.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"gopkg.in/src-d/go-errors.v1"
1515
"gopkg.in/src-d/go-git.v4"
1616
"gopkg.in/src-d/go-git.v4/config"
17+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1718
"gopkg.in/src-d/go-git.v4/storage"
1819
"gopkg.in/src-d/go-git.v4/storage/filesystem"
1920
)
@@ -142,7 +143,9 @@ func (l *Library) repositoryStorer(url *lookout.RepositoryInfo) (
142143
return nil, err
143144
}
144145

145-
return filesystem.NewStorage(fs)
146+
// TODO(carlosms) take cache size from config
147+
cache := cache.NewObjectLRU(cache.DefaultMaxSize)
148+
return filesystem.NewStorage(fs, cache), nil
146149
}
147150

148151
func (l *Library) repositoryPath(url *lookout.RepositoryInfo) string {

0 commit comments

Comments
 (0)