-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add google-storage-go buffer reuse patch
This patch adds the hability to specify a user-defined buffer to be used for file uploads to Google Storage in order to reduce memory allocations. The user needs to keep track of this buffer somehow, normally with sync.Buffer, and ensure it's not used by multiple goroutines. This patch depends on this one from googleapis: googleapis/google-api-go-client#632
- Loading branch information
Showing
1 changed file
with
100 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| diff --git a/go.mod b/go.mod | ||
| index cfce39ebd..612562de4 100644 | ||
| --- a/go.mod | ||
| +++ b/go.mod | ||
| @@ -13,11 +13,14 @@ require ( | ||
| github.com/jstemmer/go-junit-report v0.9.1 | ||
| go.opencensus.io v0.22.5 | ||
| golang.org/x/lint v0.0.0-20200302205851-738671d3881b | ||
| + golang.org/x/mod v0.4.0 // indirect | ||
| golang.org/x/net v0.0.0-20201110031124-69a78807bb2b | ||
| golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 | ||
| golang.org/x/text v0.3.4 | ||
| - golang.org/x/tools v0.0.0-20201201171514-ae829c33831c | ||
| + golang.org/x/tools v0.0.0-20201202100533-7534955ac86b | ||
| google.golang.org/api v0.35.0 | ||
| google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e | ||
| google.golang.org/grpc v1.33.2 | ||
| ) | ||
| + | ||
| +replace google.golang.org/api/googleapi => github.com/zimnx/google-api-go-client v0.30.1-0.20200901093059-b307dab300e2 | ||
| diff --git a/go.sum b/go.sum | ||
| index 28183d568..6ce911c1d 100644 | ||
| --- a/go.sum | ||
| +++ b/go.sum | ||
| @@ -174,6 +174,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de | ||
| github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= | ||
| github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= | ||
| github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= | ||
| +github.com/zimnx/google-api-go-client v0.30.1-0.20200901093059-b307dab300e2 h1:0YXnr0K0n7tgcAHtgWjSaOnkx7cxXsnIxcPpahdiaKg= | ||
| +github.com/zimnx/google-api-go-client v0.30.1-0.20200901093059-b307dab300e2/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= | ||
| go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= | ||
| go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= | ||
| go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= | ||
| @@ -232,6 +234,8 @@ golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= | ||
| golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= | ||
| golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= | ||
| golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= | ||
| +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= | ||
| +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= | ||
| golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
| golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
| golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
| @@ -331,6 +335,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zr | ||
| golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= | ||
| golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= | ||
| golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= | ||
| @@ -398,9 +403,12 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc | ||
| golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d h1:szSOL78iTCl0LF1AMjhSWJj8tIM0KixlUUnBtYXsmd8= | ||
| golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= | ||
| golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= | ||
| +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= | ||
| golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= | ||
| golang.org/x/tools v0.0.0-20201201171514-ae829c33831c h1:QhVTFg9pX+Ew8jStOfhGgUuW2rvaGPgkTWPsHNQkk7M= | ||
| golang.org/x/tools v0.0.0-20201201171514-ae829c33831c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= | ||
| +golang.org/x/tools v0.0.0-20201202100533-7534955ac86b h1:nOM4+lFhnC6uxSrRnxjZ4Azu1bF9DQz5rAsb3LUErhc= | ||
| +golang.org/x/tools v0.0.0-20201202100533-7534955ac86b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= | ||
| golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= | ||
| @@ -481,6 +489,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D | ||
| google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c h1:Lq4llNryJoaVFRmvrIwC/ZHH7tNt4tUYIu8+se2aayY= | ||
| google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= | ||
| google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= | ||
| +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= | ||
| google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= | ||
| google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e h1:wYR00/Ht+i/79g/gzhdehBgLIJCklKoc8Q/NebdzzpY= | ||
| google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= | ||
| diff --git a/storage/writer.go b/storage/writer.go | ||
| index 1843a8141..900657f53 100644 | ||
| --- a/storage/writer.go | ||
| +++ b/storage/writer.go | ||
| @@ -63,6 +63,14 @@ type Writer struct { | ||
| // ChunkSize must be set before the first Write call. | ||
| ChunkSize int | ||
|
|
||
| + // Buffer is the temporary buffer used for chunk copying. | ||
| + // This allows users to specify their own buffers in order to avoid allocations. | ||
| + // If the buffer is nil, empty or too small a new buffer of ChunkSize size will | ||
| + // be allocated for each upload. | ||
| + // | ||
| + // Buffer should have capacity >= ChunkSize. | ||
| + Buffer []byte | ||
| + | ||
| // ProgressFunc can be used to monitor the progress of a large write. | ||
| // operation. If ProgressFunc is not nil and writing requires multiple | ||
| // calls to the underlying service (see | ||
| @@ -111,6 +119,9 @@ func (w *Writer) open() error { | ||
| mediaOpts := []googleapi.MediaOption{ | ||
| googleapi.ChunkSize(w.ChunkSize), | ||
| } | ||
| + if w.Buffer != nil { | ||
| + mediaOpts = append(mediaOpts, googleapi.WithBuffer(w.Buffer)) | ||
| + } | ||
| if c := attrs.ContentType; c != "" { | ||
| mediaOpts = append(mediaOpts, googleapi.ContentType(c)) | ||
| } |