Skip to content
This repository was archived by the owner on Sep 30, 2024. It is now read-only.

codeintel: allow configuring max concurrency in uploads #56117

Merged
Strum355 merged 5 commits into from
Aug 28, 2023
Merged

codeintel: allow configuring max concurrency in uploads #56117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
acc34d9
codeintel: allow configuring max concurrency in uploads
Strum355 Aug 22, 2023
89cf675
bazel configure
Strum355 Aug 22, 2023
036dd73
actually use the ctx passed in
Strum355 Aug 24, 2023
d2cc8e7
go mod tidy
Strum355 Aug 24, 2023
3a44a08
go mod tidy 2
Strum355 Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/scim2/filter-parser/v2 v2.2.0
github.com/snabb/diagio v1.0.0 // indirect
github.com/sourcegraph/conc v0.2.0
github.com/sourcegraph/conc v0.3.0
github.com/sourcegraph/mountinfo v0.0.0-20230106004439-7026e28cef67
github.com/sourcegraph/sourcegraph/monitoring v0.0.0-20230124144931-b2d81b1accb6
github.com/sourcegraph/zoekt v0.0.0-20230811181333-956d775e32b3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1995,8 +1995,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b h1:Mlytsllyx6d1eaKXt8urXX0YjP5Tq4UGV+BfL6Yc1aQ=
github.com/sourcegraph/alertmanager v0.21.1-0.20211110092431-863f5b1ee51b/go.mod h1:0MLTrjQI8EuVmvykEhcfr/7X0xmaDAZrqMgxIq3OXHk=
github.com/sourcegraph/conc v0.2.0 h1:96VpOCAtXDCQ8Oycz0ftHqdPyMi8w12ltN4L2noYg7s=
github.com/sourcegraph/conc v0.2.0/go.mod h1:8lmPpTLA0hsWqw4lw7wS1e694U2tMjRrc1Asvupb4QM=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded h1:QcxHhicvH6TFpSmC3vZKWbwLSHmwy72+CESqjjaIsZA=
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230624001757-345a8df15ded/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0=
github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71 h1:tsWE3F3StWvnwLnC4JWb0zX0UHY9GULQtu/aoQvLJvI=
Expand Down
1 change: 1 addition & 0 deletions lib/codeintel/upload/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ go_library(
"//lib/errors",
"//lib/output",
"@com_github_klauspost_pgzip//:pgzip",
"@com_github_sourcegraph_conc//pool",
"@com_github_sourcegraph_scip//bindings/go/scip",
"@org_golang_google_protobuf//proto",
],
Expand Down
34 changes: 12 additions & 22 deletions lib/codeintel/upload/upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"os"
"sync"

"github.com/sourcegraph/conc/pool"

"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/output"
)
Expand Down Expand Up @@ -210,18 +212,15 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo
)
defer func() { complete(err) }()

var wg sync.WaitGroup
errs := make(chan error, len(readers))

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
pool := new(pool.ErrorPool).WithFirstError().WithContext(ctx)
if opts.MaxConcurrency > 0 {
pool.WithMaxGoroutines(opts.MaxConcurrency)
}

for i, reader := range readers {
wg.Add(1)

go func(i int, reader io.ReadSeeker) {
defer wg.Done()
i, reader := i, reader

pool.Go(func(ctx context.Context) error {
// Determine size of this reader. If we're not the last reader in the slice,
// then we're the maximum payload size. Otherwise, we're whatever is left.
partReaderLen := opts.MaxPayloadSizeBytes
Expand All @@ -236,25 +235,16 @@ func uploadMultipartIndexParts(ctx context.Context, httpClient Client, opts Uplo
}

if err := uploadIndexFile(ctx, httpClient, opts, reader, partReaderLen, requestOptions, progress, retry, i, len(readers)); err != nil {
errs <- err
cancel()
return err
} else if progress != nil {
// Mark complete in case we debounced our last updates
progress.SetValue(i, 1)
}
}(i, reader)
}

wg.Wait()
close(errs)

for err := range errs {
if err != nil {
return err
}
return nil
})
}

return nil
return pool.Wait()
}

// uploadMultipartIndexFinalize performs the request to stitch the uploaded parts together and
Expand Down
1 change: 1 addition & 0 deletions lib/codeintel/upload/upload_options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type SourcegraphInstanceOptions struct {
MaxRetries int // The maximum number of retries per request
RetryInterval time.Duration // Sleep duration between retries
MaxPayloadSizeBytes int64 // The maximum number of bytes sent in a single request
MaxConcurrency int // The maximum number of concurrent uploads. Only relevant for multipart uploads
GitHubToken string // GitHub token used for auth when lsif.enforceAuth is true (optional)
GitLabToken string // GitLab token used for auth when lsif.enforceAuth is true (optional)
HTTPClient Client
Expand Down
3 changes: 2 additions & 1 deletion lib/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ require (
github.com/mitchellh/copystructure v1.2.0
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
github.com/muesli/termenv v0.12.0
github.com/sourcegraph/conc v0.3.0
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
github.com/sourcegraph/log v0.0.0-20221206163500-7d93c6ad7037
Expand Down Expand Up @@ -113,7 +114,7 @@ require (
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.2.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/mod v0.8.0 // indirect
Expand Down
7 changes: 4 additions & 3 deletions lib/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3 h1:11miag7hlORpW7ici5mL7T9PyiEsmVmf+8PFOvJ/ZrA=
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf h1:oAdWFqhStsWiiMP/vkkHiMXqFXzl1XfUNOdxKJbd6bI=
Expand Down Expand Up @@ -576,14 +578,13 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
Expand Down