Skip to content

Commit c0d85ec

Browse files
Strum355Strum355
authored
codeintel: allow configuring max concurrency in uploads (#1023)
* codeintel: allow configuring max concurrency in uploads * pin to main
1 parent 511ebf8 commit c0d85ec

File tree

4 files changed

+277
-355
lines changed

4 files changed

+277
-355
lines changed

‎cmd/src/code_intel_upload.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ func codeintelUploadOptions(out *output.Output, isSCIPAvailable bool) upload.Upl
172172
RetryInterval: time.Second,
173173
Path: path,
174174
MaxPayloadSizeBytes: codeintelUploadFlags.maxPayloadSizeMb * 1000 * 1000,
175+
MaxConcurrency: codeintelUploadFlags.maxConcurrency,
175176
GitHubToken: codeintelUploadFlags.gitHubToken,
176177
GitLabToken: codeintelUploadFlags.gitLabToken,
177178
},

‎cmd/src/code_intel_upload_flags.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ var codeintelUploadFlags struct {
3636
// SourcegraphInstanceOptions
3737
uploadRoute string
3838
maxPayloadSizeMb int64
39+
maxConcurrency int
3940

4041
// Codehost authorization secrets
4142
gitHubToken string
@@ -77,6 +78,7 @@ func init() {
7778
// SourcegraphInstanceOptions
7879
codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.uploadRoute, "upload-route", "/.api/lsif/upload", "The path of the upload route. For internal use only.")
7980
codeintelUploadFlagSet.Int64Var(&codeintelUploadFlags.maxPayloadSizeMb, "max-payload-size", 100, `The maximum upload size (in megabytes). Indexes exceeding this limit will be uploaded over multiple HTTP requests.`)
81+
codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.maxConcurrency, "max-concurrency", -1, "The maximum number of concurrent uploads. Only relevant for multipart uploads. Defaults to all parts concurrently.")
8082

8183
// Codehost authorization secrets
8284
codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.gitHubToken, "github-token", "", `A GitHub access token with 'public_repo' scope that Sourcegraph uses to verify you have access to the repository.`)

‎go.mod‎

Lines changed: 61 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/sourcegraph/src-cli
33
go 1.18
44

55
require (
6-
cloud.google.com/go/storage v1.28.0
6+
cloud.google.com/go/storage v1.30.1
77
github.com/atotto/clipboard v0.1.4
88
github.com/aws/aws-sdk-go-v2/config v1.18.13
99
github.com/aws/aws-sdk-go-v2/service/ec2 v1.86.0
@@ -12,30 +12,30 @@ require (
1212
github.com/charmbracelet/bubbles v0.15.0
1313
github.com/charmbracelet/lipgloss v0.7.1
1414
github.com/creack/goselect v0.1.2
15-
github.com/derision-test/glock v0.0.0-20210316032053-f5b74334bb29
15+
github.com/derision-test/glock v1.0.0
1616
github.com/dineshappavoo/basex v0.0.0-20170425072625-481a6f6dc663
1717
github.com/dustin/go-humanize v1.0.1
1818
github.com/gobwas/glob v0.2.3
1919
github.com/google/go-cmp v0.5.9
2020
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
21-
github.com/hexops/autogold v1.3.0
21+
github.com/hexops/autogold v1.3.1
2222
github.com/jedib0t/go-pretty/v6 v6.3.7
2323
github.com/jig/teereadcloser v0.0.0-20181016160506-953720c48e05
2424
github.com/json-iterator/go v1.1.12
2525
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
26-
github.com/mattn/go-isatty v0.0.17
26+
github.com/mattn/go-isatty v0.0.19
2727
github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c
2828
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
2929
github.com/sourcegraph/conc v0.3.0
3030
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3
3131
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
32-
github.com/sourcegraph/scip v0.2.4-0.20221213205653-aa0e511dcfef
33-
github.com/sourcegraph/sourcegraph/lib v0.0.0-20230316093010-26299ec302d0
34-
github.com/stretchr/testify v1.8.1
35-
golang.org/x/net v0.8.0
36-
golang.org/x/sync v0.1.0
37-
google.golang.org/api v0.110.0
38-
google.golang.org/protobuf v1.28.1
32+
github.com/sourcegraph/scip v0.3.1-0.20230627154934-45df7f6d33fc
33+
github.com/sourcegraph/sourcegraph/lib v0.0.0-20230822113036-acc34d9b2e95
34+
github.com/stretchr/testify v1.8.4
35+
golang.org/x/net v0.12.0
36+
golang.org/x/sync v0.3.0
37+
google.golang.org/api v0.132.0
38+
google.golang.org/protobuf v1.31.0
3939
gopkg.in/yaml.v3 v3.0.1
4040
jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7
4141
k8s.io/api v0.27.1
@@ -45,10 +45,26 @@ require (
4545
)
4646

4747
require (
48-
cloud.google.com/go v0.107.0 // indirect
49-
cloud.google.com/go/compute v1.18.0 // indirect
50-
cloud.google.com/go/compute/metadata v0.2.3 // indirect
51-
cloud.google.com/go/iam v0.12.0 // indirect
48+
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
49+
github.com/google/s2a-go v0.1.4 // indirect
50+
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
51+
github.com/jackc/pgconn v1.12.1 // indirect
52+
github.com/jackc/pgio v1.0.0 // indirect
53+
github.com/jackc/pgpassfile v1.0.0 // indirect
54+
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
55+
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
56+
github.com/onsi/ginkgo/v2 v2.9.7 // indirect
57+
github.com/onsi/gomega v1.27.8 // indirect
58+
go.uber.org/goleak v1.2.1 // indirect
59+
google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753 // indirect
60+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230717213848-3f92550aa753 // indirect
61+
)
62+
63+
require (
64+
cloud.google.com/go v0.110.4 // indirect
65+
cloud.google.com/go/compute v1.22.0 // indirect
66+
cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect
67+
cloud.google.com/go/iam v1.1.1 // indirect
5268
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
5369
github.com/Masterminds/goutils v1.1.1 // indirect
5470
github.com/Masterminds/semver v1.5.0 // indirect
@@ -70,19 +86,19 @@ require (
7086
github.com/bufbuild/buf v1.4.0 // indirect
7187
github.com/charmbracelet/bubbletea v0.23.1 // direct
7288
github.com/charmbracelet/glamour v0.5.0 // indirect
73-
github.com/cockroachdb/errors v1.9.0 // indirect
74-
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
75-
github.com/cockroachdb/redact v1.1.3 // indirect
89+
github.com/cockroachdb/errors v1.10.0 // indirect
90+
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
91+
github.com/cockroachdb/redact v1.1.5 // indirect
7692
github.com/containerd/console v1.0.3 // indirect
7793
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
7894
github.com/davecgh/go-spew v1.1.1 // indirect
79-
github.com/dlclark/regexp2 v1.7.0 // indirect
95+
github.com/dlclark/regexp2 v1.8.0 // indirect
8096
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
81-
github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect
82-
github.com/fatih/color v1.13.0 // indirect
83-
github.com/getsentry/sentry-go v0.15.0 // indirect
97+
github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect
98+
github.com/fatih/color v1.15.0 // indirect
99+
github.com/getsentry/sentry-go v0.22.0 // indirect
84100
github.com/ghodss/yaml v1.0.0 // indirect
85-
github.com/go-logr/logr v1.2.3 // indirect
101+
github.com/go-logr/logr v1.2.4 // indirect
86102
github.com/go-openapi/jsonpointer v0.19.6 // indirect
87103
github.com/go-openapi/jsonreference v0.20.1 // indirect
88104
github.com/go-openapi/swag v0.22.3 // indirect
@@ -92,22 +108,22 @@ require (
92108
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
93109
github.com/golang/protobuf v1.5.3 // indirect
94110
github.com/google/gnostic v0.5.7-v3refs // indirect
95-
github.com/google/gofuzz v1.1.0 // indirect
111+
github.com/google/gofuzz v1.2.0 // indirect
96112
github.com/google/uuid v1.3.0 // indirect
97-
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
98-
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
113+
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
114+
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
99115
github.com/gorilla/css v1.0.0 // indirect
100116
github.com/hexops/gotextdiff v1.0.3 // indirect
101117
github.com/hexops/valast v1.4.3 // indirect
102118
github.com/huandu/xstrings v1.3.2 // indirect
103-
github.com/imdario/mergo v0.3.13 // indirect
104-
github.com/inconshreveable/mousetrap v1.0.1 // indirect
119+
github.com/imdario/mergo v0.3.16 // indirect
120+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
105121
github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
106122
github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect
107123
github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect
108124
github.com/jmespath/go-jmespath v0.4.0 // indirect
109125
github.com/josharian/intern v1.0.0 // indirect
110-
github.com/klauspost/compress v1.15.11 // indirect
126+
github.com/klauspost/compress v1.16.7 // indirect
111127
github.com/klauspost/pgzip v1.2.5 // indirect
112128
github.com/kr/pretty v0.3.1 // indirect
113129
github.com/kr/text v0.2.0 // indirect
@@ -116,7 +132,7 @@ require (
116132
github.com/mattn/go-colorable v0.1.13 // indirect
117133
github.com/mattn/go-localereader v0.0.1 // indirect
118134
github.com/mattn/go-runewidth v0.0.14 // indirect
119-
github.com/microcosm-cc/bluemonday v1.0.22 // indirect
135+
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
120136
github.com/mitchellh/copystructure v1.2.0 // indirect
121137
github.com/mitchellh/reflectwalk v1.0.2 // indirect
122138
github.com/moby/spdystream v0.2.0 // indirect
@@ -136,11 +152,11 @@ require (
136152
github.com/pmezard/go-difflib v1.0.0 // indirect
137153
github.com/pseudomuto/protoc-gen-doc v1.5.1 // indirect
138154
github.com/pseudomuto/protokit v0.2.0 // indirect
139-
github.com/rivo/uniseg v0.4.2 // indirect
155+
github.com/rivo/uniseg v0.4.3 // indirect
140156
github.com/rogpeppe/go-internal v1.10.0 // indirect
141157
github.com/russross/blackfriday/v2 v2.1.0 // indirect
142-
github.com/sourcegraph/log v0.0.0-20221206163500-7d93c6ad7037 // indirect
143-
github.com/spf13/cobra v1.6.0 // indirect
158+
github.com/sourcegraph/log v0.0.0-20230711093019-40c57b632cca // indirect
159+
github.com/spf13/cobra v1.7.0 // indirect
144160
github.com/spf13/pflag v1.0.5 // indirect
145161
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
146162
github.com/stretchr/objx v0.5.0 // indirect
@@ -150,21 +166,21 @@ require (
150166
github.com/yuin/goldmark v1.5.2 // indirect
151167
github.com/yuin/goldmark-emoji v1.0.1 // indirect
152168
go.opencensus.io v0.24.0 // indirect
153-
go.uber.org/atomic v1.10.0 // indirect
154-
go.uber.org/multierr v1.9.0 // indirect
169+
go.uber.org/atomic v1.11.0 // indirect
170+
go.uber.org/multierr v1.11.0 // indirect
155171
go.uber.org/zap v1.24.0 // indirect
156-
golang.org/x/crypto v0.3.0 // indirect
157-
golang.org/x/mod v0.9.0 // indirect
158-
golang.org/x/oauth2 v0.5.0 // indirect
159-
golang.org/x/sys v0.6.0 // indirect
160-
golang.org/x/term v0.6.0 // indirect
161-
golang.org/x/text v0.8.0 // indirect
162-
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
163-
golang.org/x/tools v0.7.0 // indirect
172+
golang.org/x/crypto v0.11.0 // indirect
173+
golang.org/x/mod v0.11.0 // indirect
174+
golang.org/x/oauth2 v0.10.0 // indirect
175+
golang.org/x/sys v0.10.0 // indirect
176+
golang.org/x/term v0.10.0 // indirect
177+
golang.org/x/text v0.11.0 // indirect
178+
golang.org/x/time v0.3.0 // indirect
179+
golang.org/x/tools v0.10.0 // indirect
164180
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
165181
google.golang.org/appengine v1.6.7 // indirect
166-
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
167-
google.golang.org/grpc v1.53.0 // indirect
182+
google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753 // indirect
183+
google.golang.org/grpc v1.56.2 // indirect
168184
gopkg.in/inf.v0 v0.9.1 // direct
169185
gopkg.in/yaml.v2 v2.4.0 // indirect
170186
k8s.io/klog/v2 v2.90.1 // indirect

0 commit comments

Comments
 (0)