From 0aa4a48d9a61260276cbf1a72ba4559f9a137cdc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 09:05:39 +0000 Subject: [PATCH] Bump cloud.google.com/go/storage from 1.18.2 to 1.19.0 Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.18.2 to 1.19.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/storage/v1.18.2...spanner/v1.19.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 8 +- vendor/cloud.google.com/go/storage/CHANGES.md | 32 + vendor/cloud.google.com/go/storage/README.md | 2 +- vendor/cloud.google.com/go/storage/acl.go | 38 +- vendor/cloud.google.com/go/storage/bucket.go | 189 ++- vendor/cloud.google.com/go/storage/copy.go | 14 +- vendor/cloud.google.com/go/storage/doc.go | 52 +- .../go/storage/emulator_test.sh | 71 + vendor/cloud.google.com/go/storage/hmac.go | 33 +- vendor/cloud.google.com/go/storage/iam.go | 15 +- .../go/storage/internal/apiv2/doc.go | 20 +- .../internal/apiv2/gapic_metadata.json | 125 ++ .../storage/internal/apiv2/storage_client.go | 1376 ++++++++++++++++- vendor/cloud.google.com/go/storage/invoke.go | 54 +- .../go/storage/notifications.go | 15 +- .../go/storage/post_policy_v4.go | 99 +- vendor/cloud.google.com/go/storage/reader.go | 12 +- vendor/cloud.google.com/go/storage/storage.go | 224 ++- vendor/cloud.google.com/go/storage/writer.go | 16 + vendor/modules.txt | 2 +- 21 files changed, 2229 insertions(+), 170 deletions(-) create mode 100644 vendor/cloud.google.com/go/storage/emulator_test.sh diff --git a/go.mod b/go.mod index 8edcf2d90e..b3d77661e9 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace ( ) require ( - cloud.google.com/go/storage v1.18.2 + cloud.google.com/go/storage v1.19.0 github.com/Azure/azure-storage-blob-go v0.14.0 github.com/aws/aws-sdk-go v1.42.44 github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20211215200129-69c85dc22db6 diff --git a/go.sum b/go.sum index 139a7ce670..487f39b294 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.18.2 h1:5NQw6tOn3eMm0oE8vTkfjau18kjL79FlMjy/CHTpmoY= -cloud.google.com/go/storage v1.18.2/go.mod h1:AiIj7BWXyhO5gGVmYJ+S8tbkCx3yb0IMjua8Aw4naVM= +cloud.google.com/go/storage v1.19.0 h1:XOQSnPJD8hRtZJ3VdCyK0mBZsGGImrzPAMbSWcHSe6Q= +cloud.google.com/go/storage v1.19.0/go.mod h1:6rgiTRjOqI/Zd9YKimub5TIB4d+p3LH33V3ZE1DMuUM= code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= @@ -1746,7 +1746,6 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1920,7 +1919,6 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6 google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= @@ -2006,10 +2004,8 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211016002631-37fc39342514/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= diff --git a/vendor/cloud.google.com/go/storage/CHANGES.md b/vendor/cloud.google.com/go/storage/CHANGES.md index 6c0ef20999..d804caa34a 100644 --- a/vendor/cloud.google.com/go/storage/CHANGES.md +++ b/vendor/cloud.google.com/go/storage/CHANGES.md @@ -1,5 +1,37 @@ # Changes +## [1.19.0](https://www.github.com/googleapis/google-cloud-go/compare/storage/v1.18.2...storage/v1.19.0) (2022-01-25) + + +### Features + +* **storage:** add fully configurable and idempotency-aware retry strategy ([#5384](https://www.github.com/googleapis/google-cloud-go/issues/5384), [#5185](https://www.github.com/googleapis/google-cloud-go/issues/5185), [#5170](https://www.github.com/googleapis/google-cloud-go/issues/5170), [#5223](https://www.github.com/googleapis/google-cloud-go/issues/5223), [#5221](https://www.github.com/googleapis/google-cloud-go/issues/5221), [#5193](https://www.github.com/googleapis/google-cloud-go/issues/5193), [#5159](https://www.github.com/googleapis/google-cloud-go/issues/5159), [#5165](https://www.github.com/googleapis/google-cloud-go/issues/5165), [#5166](https://www.github.com/googleapis/google-cloud-go/issues/5166), [#5210](https://www.github.com/googleapis/google-cloud-go/issues/5210), [#5172](https://www.github.com/googleapis/google-cloud-go/issues/5172), [#5314](https://www.github.com/googleapis/google-cloud-go/issues/5314)) + * This release contains changes to fully align this library's retry strategy + with best practices as described in the + Cloud Storage [docs](https://cloud.google.com/storage/docs/retry-strategy). + * The library will now retry only idempotent operations by default. This means + that for certain operations, including object upload, compose, rewrite, + update, and delete, requests will not be retried by default unless + [idempotency conditions](https://cloud.google.com/storage/docs/retry-strategy#idempotency) + for the request have been met. + * The library now has methods to configure aspects of retry policy for + API calls, including which errors are retried, the timing of the + exponential backoff, and how idempotency is taken into account. + * If you wish to re-enable retries for a non-idempotent request, use the + [RetryAlways](https://pkg.go.dev/cloud.google.com/go/storage@main#RetryAlways) + policy. + * For full details on how to configure retries, see the + [package docs](https://pkg.go.dev/cloud.google.com/go/storage@main#hdr-Retrying_failed_requests) + and the + [Cloud Storage docs](https://cloud.google.com/storage/docs/retry-strategy) +* **storage:** GenerateSignedPostPolicyV4 can use existing creds to authenticate ([#5105](https://www.github.com/googleapis/google-cloud-go/issues/5105)) ([46489f4](https://www.github.com/googleapis/google-cloud-go/commit/46489f4c8a634068a3e7cf2fd5e5ca11b555c0a8)) +* **storage:** post policy can be signed with a fn that takes raw bytes ([#5079](https://www.github.com/googleapis/google-cloud-go/issues/5079)) ([25d1278](https://www.github.com/googleapis/google-cloud-go/commit/25d1278cab539fbfdd8563ed6b297e30d3fe555c)) +* **storage:** add rpo (turbo replication) support ([#5003](https://www.github.com/googleapis/google-cloud-go/issues/5003)) ([3bd5995](https://www.github.com/googleapis/google-cloud-go/commit/3bd59958e0c06d2655b67fcb5410668db3c52af0)) + +### Bug Fixes + +* **storage:** fix nil check in gRPC Reader ([#5376](https://www.github.com/googleapis/google-cloud-go/issues/5376)) ([5e7d722](https://www.github.com/googleapis/google-cloud-go/commit/5e7d722d18a62b28ba98169b3bdbb49401377264)) + ### [1.18.2](https://www.github.com/googleapis/google-cloud-go/compare/storage/v1.18.1...storage/v1.18.2) (2021-10-18) diff --git a/vendor/cloud.google.com/go/storage/README.md b/vendor/cloud.google.com/go/storage/README.md index 13c89ea6de..8536d9cab9 100644 --- a/vendor/cloud.google.com/go/storage/README.md +++ b/vendor/cloud.google.com/go/storage/README.md @@ -3,7 +3,7 @@ - [About Cloud Storage](https://cloud.google.com/storage/) - [API documentation](https://cloud.google.com/storage/docs) - [Go client documentation](https://pkg.go.dev/cloud.google.com/go/storage) -- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage) +- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/main/storage) ### Example Usage diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index 79b398c221..9bdf96e3ed 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -73,6 +73,7 @@ type ACLHandle struct { object string isDefault bool userProject string // for requester-pays buckets + retry *retryConfig } // Delete permanently deletes the ACL entry for the given entity. @@ -120,12 +121,12 @@ func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) { func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { req := a.c.raw.DefaultObjectAccessControls.List(a.bucket) a.configureCall(ctx, req) acls, err = req.Do() return err - }) + }, a.retry, true) if err != nil { return nil, err } @@ -135,18 +136,21 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)) a.configureCall(ctx, req) - return req.Do() + + return run(ctx, func() error { + return req.Do() + }, a.retry, false) } func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { var acls *raw.BucketAccessControls var err error - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { req := a.c.raw.BucketAccessControls.List(a.bucket) a.configureCall(ctx, req) acls, err = req.Do() return err - }) + }, a.retry, true) if err != nil { return nil, err } @@ -161,25 +165,29 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol } req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl) a.configureCall(ctx, req) - _, err := req.Do() - return err + return run(ctx, func() error { + _, err := req.Do() + return err + }, a.retry, false) } func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)) a.configureCall(ctx, req) - return req.Do() + return run(ctx, func() error { + return req.Do() + }, a.retry, false) } func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object) a.configureCall(ctx, req) acls, err = req.Do() return err - }) + }, a.retry, true) if err != nil { return nil, err } @@ -204,14 +212,18 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl) } a.configureCall(ctx, req) - _, err := req.Do() - return err + return run(ctx, func() error { + _, err := req.Do() + return err + }, a.retry, false) } func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)) a.configureCall(ctx, req) - return req.Do() + return run(ctx, func() error { + return req.Do() + }, a.retry, false) } func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) { diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index ec7dcb5c32..9d145f0396 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -44,6 +44,7 @@ type BucketHandle struct { defaultObjectACL ACLHandle conds *BucketConditions userProject string // project for Requester Pays buckets + retry *retryConfig } // Bucket returns a BucketHandle, which provides operations on the named bucket. @@ -54,18 +55,22 @@ type BucketHandle struct { // found at: // https://cloud.google.com/storage/docs/bucket-naming func (c *Client) Bucket(name string) *BucketHandle { + retry := c.retry.clone() return &BucketHandle{ c: c, name: name, acl: ACLHandle{ c: c, bucket: name, + retry: retry, }, defaultObjectACL: ACLHandle{ c: c, bucket: name, isDefault: true, + retry: retry, }, + retry: retry, } } @@ -95,7 +100,7 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck if attrs != nil && attrs.PredefinedDefaultObjectACL != "" { req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL) } - return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) + return run(ctx, func() error { _, err := req.Context(ctx).Do(); return err }, b.retry, true) } // Delete deletes the Bucket. @@ -107,7 +112,8 @@ func (b *BucketHandle) Delete(ctx context.Context) (err error) { if err != nil { return err } - return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) + + return run(ctx, func() error { return req.Context(ctx).Do() }, b.retry, true) } func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) { @@ -144,6 +150,7 @@ func (b *BucketHandle) DefaultObjectACL() *ACLHandle { // for valid object names can be found at: // https://cloud.google.com/storage/docs/naming-objects func (b *BucketHandle) Object(name string) *ObjectHandle { + retry := b.retry.clone() return &ObjectHandle{ c: b.c, bucket: b.name, @@ -153,9 +160,11 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { bucket: b.name, object: name, userProject: b.userProject, + retry: retry, }, gen: -1, userProject: b.userProject, + retry: retry, } } @@ -169,10 +178,10 @@ func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error return nil, err } var resp *raw.Bucket - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { resp, err = req.Context(ctx).Do() return err - }) + }, b.retry, true) var e *googleapi.Error if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { return nil, ErrBucketNotExist @@ -210,12 +219,20 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( if uattrs.PredefinedDefaultObjectACL != "" { req.PredefinedDefaultObjectAcl(uattrs.PredefinedDefaultObjectACL) } - // TODO(jba): retry iff metagen is set? - rb, err := req.Context(ctx).Do() - if err != nil { + + isIdempotent := b.conds != nil && b.conds.MetagenerationMatch != 0 + + var rawBucket *raw.Bucket + call := func() error { + rb, err := req.Context(ctx).Do() + rawBucket = rb + return err + } + + if err := run(ctx, call, b.retry, isIdempotent); err != nil { return nil, err } - return newBucket(rb) + return newBucket(rawBucket) } func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) { @@ -282,8 +299,54 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, return SignedURL(b.name, object, newopts) } -// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to -// omit PrivateKey/SignBytes +// GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts. +// The generated URL and fields will then allow an unauthenticated client to perform multipart uploads. +// +// This method only requires the Expires field in the specified PostPolicyV4Options +// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey +// from the GOOGLE_APPLICATION_CREDENTIALS environment variable. +// If you are authenticating with a custom HTTP client, Service Account based +// auto-detection will be hindered. +// +// If no private key is found, it attempts to use the GoogleAccessID to sign the URL. +// This requires the IAM Service Account Credentials API to be enabled +// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview) +// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// If you do not want these fields set for you, you may pass them in through opts or use +// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead. +func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { + if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) { + return GenerateSignedPostPolicyV4(b.name, object, opts) + } + // Make a copy of opts so we don't modify the pointer parameter. + newopts := opts.clone() + + if newopts.GoogleAccessID == "" { + id, err := b.detectDefaultGoogleAccessID() + if err != nil { + return nil, err + } + newopts.GoogleAccessID = id + } + if newopts.SignBytes == nil && newopts.SignRawBytes == nil && len(newopts.PrivateKey) == 0 { + if b.c.creds != nil && len(b.c.creds.JSON) > 0 { + var sa struct { + PrivateKey string `json:"private_key"` + } + err := json.Unmarshal(b.c.creds.JSON, &sa) + if err == nil && sa.PrivateKey != "" { + newopts.PrivateKey = []byte(sa.PrivateKey) + } + } + + // Don't error out if we can't unmarshal the private key from the client, + // fallback to the default sign function for the service account. + if len(newopts.PrivateKey) == 0 { + newopts.SignRawBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID) + } + } + return GenerateSignedPostPolicyV4(b.name, object, newopts) +} func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)") @@ -461,6 +524,12 @@ type BucketAttrs struct { // The project number of the project the bucket belongs to. // This field is read-only. ProjectNumber uint64 + + // RPO configures the Recovery Point Objective (RPO) policy of the bucket. + // Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket. + // See https://cloud.google.com/storage/docs/managing-turbo-replication for + // more information. + RPO RPO } // BucketPolicyOnly is an alias for UniformBucketLevelAccess. @@ -728,6 +797,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) { Etag: b.Etag, LocationType: b.LocationType, ProjectNumber: b.ProjectNumber, + RPO: toRPO(b), }, nil } @@ -780,6 +850,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { Logging: b.Logging.toRawBucketLogging(), Website: b.Website.toRawBucketWebsite(), IamConfiguration: bktIAM, + Rpo: b.RPO.String(), } } @@ -889,6 +960,12 @@ type BucketAttrsToUpdate struct { // See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch. PredefinedDefaultObjectACL string + // RPO configures the Recovery Point Objective (RPO) policy of the bucket. + // Set to RPOAsyncTurbo to turn on Turbo Replication for a bucket. + // See https://cloud.google.com/storage/docs/managing-turbo-replication for + // more information. + RPO RPO + setLabels map[string]string deleteLabels map[string]bool } @@ -1001,7 +1078,10 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { rb.DefaultObjectAcl = nil rb.ForceSendFields = append(rb.ForceSendFields, "DefaultObjectAcl") } + rb.StorageClass = ua.StorageClass + rb.Rpo = ua.RPO.String() + if ua.setLabels != nil || ua.deleteLabels != nil { rb.Labels = map[string]string{} for k, v := range ua.setLabels { @@ -1081,10 +1161,10 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error { metageneration = b.conds.MetagenerationMatch } req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration) - return runWithRetry(ctx, func() error { + return run(ctx, func() error { _, err := req.Context(ctx).Do() return err - }) + }, b.retry, true) } // applyBucketConds modifies the provided call using the conditions in conds. @@ -1347,6 +1427,20 @@ func toPublicAccessPrevention(b *raw.BucketIamConfiguration) PublicAccessPrevent } } +func toRPO(b *raw.Bucket) RPO { + if b == nil { + return RPOUnknown + } + switch b.Rpo { + case rpoDefault: + return RPODefault + case rpoAsyncTurbo: + return RPOAsyncTurbo + default: + return RPOUnknown + } +} + // Objects returns an iterator over the objects in the bucket that match the // Query q. If q is nil, no filtering is done. Objects will be iterated over // lexicographically by name. @@ -1367,6 +1461,33 @@ func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { return it } +// Retryer returns a bucket handle that is configured with custom retry +// behavior as specified by the options that are passed to it. All operations +// on the new handle will use the customized retry configuration. +// Retry options set on a object handle will take precedence over options set on +// the bucket handle. +// These retry options will merge with the client's retry configuration (if set) +// for the returned handle. Options passed into this method will take precedence +// over retry options on the client. Note that you must explicitly pass in each +// option you want to override. +func (b *BucketHandle) Retryer(opts ...RetryOption) *BucketHandle { + b2 := *b + var retry *retryConfig + if b.retry != nil { + // merge the options with the existing retry + retry = b.retry + } else { + retry = &retryConfig{} + } + for _, opt := range opts { + opt.apply(retry) + } + b2.retry = retry + b2.acl.retry = retry + b2.defaultObjectACL.retry = retry + return &b2 +} + // An ObjectIterator is an iterator over ObjectAttrs. // // Note: This iterator is not safe for concurrent operations without explicit synchronization. @@ -1434,10 +1555,10 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) } var resp *raw.Objects var err error - err = runWithRetry(it.ctx, func() error { + err = run(it.ctx, func() error { resp, err = req.Context(it.ctx).Do() return err - }) + }, it.bucket.retry, true) if err != nil { var e *googleapi.Error if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { @@ -1518,10 +1639,10 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e req.MaxResults(int64(pageSize)) } var resp *raw.Buckets - err = runWithRetry(it.ctx, func() error { + err = run(it.ctx, func() error { resp, err = req.Context(it.ctx).Do() return err - }) + }, it.client.retry, true) if err != nil { return "", err } @@ -1534,3 +1655,39 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, e } return resp.NextPageToken, nil } + +// RPO (Recovery Point Objective) configures the turbo replication feature. See +// https://cloud.google.com/storage/docs/managing-turbo-replication for more information. +type RPO int + +const ( + // RPOUnknown is a zero value. It may be returned from bucket.Attrs() if RPO + // is not present in the bucket metadata, that is, the bucket is not dual-region. + // This value is also used if the RPO field is not set in a call to GCS. + RPOUnknown RPO = iota + + // RPODefault represents default replication. It is used to reset RPO on an + // existing bucket that has this field set to RPOAsyncTurbo. Otherwise it + // is equivalent to RPOUnknown, and is always ignored. This value is valid + // for dual- or multi-region buckets. + RPODefault + + // RPOAsyncTurbo represents turbo replication and is used to enable Turbo + // Replication on a bucket. This value is only valid for dual-region buckets. + RPOAsyncTurbo + + rpoUnknown string = "" + rpoDefault = "DEFAULT" + rpoAsyncTurbo = "ASYNC_TURBO" +) + +func (rpo RPO) String() string { + switch rpo { + case RPODefault: + return rpoDefault + case RPOAsyncTurbo: + return rpoAsyncTurbo + default: + return rpoUnknown + } +} diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index 61983df5ad..f180753520 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -138,8 +138,11 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr var res *raw.RewriteResponse var err error setClientHeader(call.Header()) - err = runWithRetry(ctx, func() error { res, err = call.Do(); return err }) - if err != nil { + + retryCall := func() error { res, err = call.Do(); return err } + isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist) + + if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil { return nil, err } c.RewriteToken = res.RewriteToken @@ -230,8 +233,11 @@ func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) { } var obj *raw.Object setClientHeader(call.Header()) - err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) - if err != nil { + + retryCall := func() error { obj, err = call.Do(); return err } + isIdempotent := c.dst.conds != nil && (c.dst.conds.GenerationMatch != 0 || c.dst.conds.DoesNotExist) + + if err := run(ctx, retryCall, c.dst.retry, isIdempotent); err != nil { return nil, err } return newObject(obj), nil diff --git a/vendor/cloud.google.com/go/storage/doc.go b/vendor/cloud.google.com/go/storage/doc.go index 53259936d6..f90dbe905c 100644 --- a/vendor/cloud.google.com/go/storage/doc.go +++ b/vendor/cloud.google.com/go/storage/doc.go @@ -19,15 +19,9 @@ Google Cloud Storage stores data in named objects, which are grouped into bucket More information about Google Cloud Storage is available at https://cloud.google.com/storage/docs. -See https://godoc.org/cloud.google.com/go for authentication, timeouts, +See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts, connection pooling and similar aspects of this package. -All of the methods of this package use exponential backoff to retry calls that fail -with certain errors, as described in -https://cloud.google.com/storage/docs/exponential-backoff. Retrying continues -indefinitely unless the controlling context is canceled or the client is closed. See -context.WithTimeout and context.WithCancel. - Creating a Client @@ -246,12 +240,52 @@ as the documentation of GenerateSignedPostPolicyV4. Errors -Errors returned by this client are often of the type [`googleapi.Error`](https://godoc.org/google.golang.org/api/googleapi#Error). -These errors can be introspected for more information by using `errors.As` with the richer `googleapi.Error` type. For example: +Errors returned by this client are often of the type googleapi.Error. +These errors can be introspected for more information by using errors.As +with the richer googleapi.Error type. For example: var e *googleapi.Error if ok := errors.As(err, &e); ok { if e.Code == 409 { ... } } + +See https://pkg.go.dev/google.golang.org/api/googleapi#Error for more information. + +Retrying failed requests + +Methods in this package may retry calls that fail with transient errors. +Retrying continues indefinitely unless the controlling context is canceled, the +client is closed, or a non-transient error is received. To stop retries from +continuing, use context timeouts or cancellation. + +The retry strategy in this library follows best practices for Cloud Storage. By +default, operations are retried only if they are idempotent, and exponential +backoff with jitter is employed. In addition, errors are only retried if they +are defined as transient by the service. See +https://cloud.google.com/storage/docs/retry-strategy for more information. + +Users can configure non-default retry behavior for a single library call (using +BucketHandle.Retryer and ObjectHandle.Retryer) or for all calls made by a +client (using Client.SetRetry). For example: + + o := client.Bucket(bucket).Object(object).Retryer( + // Use WithBackoff to change the timing of the exponential backoff. + storage.WithBackoff(gax.Backoff{ + Initial: 2 * time.Second, + }), + // Use WithPolicy to configure the idempotency policy. RetryAlways will + // retry the operation even if it is non-idempotent. + storage.WithPolicy(storage.RetryAlways), + ) + + // Use a context timeout to set an overall deadline on the call, including all + // potential retries. + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + // Delete an object using the specified strategy and timeout. + if err := o.Delete(ctx); err != nil { + // Handle err. + } */ package storage // import "cloud.google.com/go/storage" diff --git a/vendor/cloud.google.com/go/storage/emulator_test.sh b/vendor/cloud.google.com/go/storage/emulator_test.sh new file mode 100644 index 0000000000..e1cf4af76a --- /dev/null +++ b/vendor/cloud.google.com/go/storage/emulator_test.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.. + +# Fail on any error +set -eo pipefail + +# Display commands being run +set -x + +# Only run on Go 1.17+ +min_minor_ver=17 + +v=`go version | { read _ _ v _; echo ${v#go}; }` +comps=(${v//./ }) +minor_ver=${comps[1]} + +if [ "$minor_ver" -lt "$min_minor_ver" ]; then + echo minor version $minor_ver, skipping + exit 0 +fi + +export STORAGE_EMULATOR_HOST="http://localhost:9000" + +DEFAULT_IMAGE_NAME='gcr.io/cloud-devrel-public-resources/storage-testbench' +DEFAULT_IMAGE_TAG='latest' +DOCKER_IMAGE=${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG} +CONTAINER_NAME=storage_testbench + +# Get the docker image for the testbench +docker pull $DOCKER_IMAGE + +# Start the testbench +# Note: --net=host makes the container bind directly to the Docker host’s network, +# with no network isolation. If we were to use port-mapping instead, reset connection errors +# would be captured differently and cause unexpected test behaviour. +# The host networking driver works only on Linux hosts. +# See more about using host networking: https://docs.docker.com/network/host/ +docker run --name $CONTAINER_NAME --rm --net=host $DOCKER_IMAGE & +echo "Running the Cloud Storage testbench: $STORAGE_EMULATOR_HOST" + +# Check that the server is running - retry several times to allow for start-up time +response=$(curl -w "%{http_code}\n" $STORAGE_EMULATOR_HOST --retry-connrefused --retry 5 -o /dev/null) + +if [[ $response != 200 ]] +then + echo "Testbench server did not start correctly" + exit 1 +fi + +# Stop the testbench & cleanup environment variables +function cleanup() { + echo "Cleanup testbench" + docker stop $CONTAINER_NAME + unset STORAGE_EMULATOR_HOST; +} +trap cleanup EXIT + +# Run tests +go test -v -timeout 10m ./ -run="TestRetryConformance" -short 2>&1 | tee -a sponge_log.log diff --git a/vendor/cloud.google.com/go/storage/hmac.go b/vendor/cloud.google.com/go/storage/hmac.go index 34b6d1fb57..6f834d4e9e 100644 --- a/vendor/cloud.google.com/go/storage/hmac.go +++ b/vendor/cloud.google.com/go/storage/hmac.go @@ -89,8 +89,8 @@ type HMACKey struct { type HMACKeyHandle struct { projectID string accessID string - - raw *raw.ProjectsHmacKeysService + retry *retryConfig + raw *raw.ProjectsHmacKeysService } // HMACKeyHandle creates a handle that will be used for HMACKey operations. @@ -100,6 +100,7 @@ func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle { return &HMACKeyHandle{ projectID: projectID, accessID: accessID, + retry: c.retry, raw: raw.NewProjectsHmacKeysService(c.raw), } } @@ -126,10 +127,10 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC var metadata *raw.HmacKeyMetadata var err error - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { metadata, err = call.Context(ctx).Do() return err - }) + }, hkh.retry, true) if err != nil { return nil, err } @@ -156,9 +157,9 @@ func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) err } setClientHeader(delCall.Header()) - return runWithRetry(ctx, func() error { + return run(ctx, func() error { return delCall.Context(ctx).Do() - }) + }, hkh.retry, true) } func pbHmacKeyToHMACKey(pb *raw.HmacKey, updatedTimeCanBeNil bool) (*HMACKey, error) { @@ -214,8 +215,13 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma setClientHeader(call.Header()) - hkPb, err := call.Context(ctx).Do() - if err != nil { + var hkPb *raw.HmacKey + + if err := run(ctx, func() error { + h, err := call.Context(ctx).Do() + hkPb = h + return err + }, c.retry, false); err != nil { return nil, err } @@ -257,10 +263,11 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt var metadata *raw.HmacKeyMetadata var err error - err = runWithRetry(ctx, func() error { + isIdempotent := len(au.Etag) > 0 + err = run(ctx, func() error { metadata, err = call.Context(ctx).Do() return err - }) + }, h.retry, isIdempotent) if err != nil { return nil, err @@ -285,6 +292,7 @@ type HMACKeysIterator struct { nextFunc func() error index int desc hmacKeyDesc + retry *retryConfig } // ListHMACKeys returns an iterator for listing HMACKeys. @@ -297,6 +305,7 @@ func (c *Client) ListHMACKeys(ctx context.Context, projectID string, opts ...HMA ctx: ctx, raw: raw.NewProjectsHmacKeysService(c.raw), projectID: projectID, + retry: c.retry, } for _, opt := range opts { @@ -361,10 +370,10 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string, ctx := it.ctx var resp *raw.HmacKeysMetadata - err = runWithRetry(it.ctx, func() error { + err = run(it.ctx, func() error { resp, err = call.Context(ctx).Do() return err - }) + }, it.retry, true) if err != nil { return "", err } diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go index 5caefb059d..a870ab8ee8 100644 --- a/vendor/cloud.google.com/go/storage/iam.go +++ b/vendor/cloud.google.com/go/storage/iam.go @@ -29,6 +29,7 @@ func (b *BucketHandle) IAM() *iam.Handle { return iam.InternalNewHandleClient(&iamClient{ raw: b.c.raw, userProject: b.userProject, + retry: b.retry, }, b.name) } @@ -36,6 +37,7 @@ func (b *BucketHandle) IAM() *iam.Handle { type iamClient struct { raw *raw.Service userProject string + retry *retryConfig } func (c *iamClient) Get(ctx context.Context, resource string) (p *iampb.Policy, err error) { @@ -52,10 +54,10 @@ func (c *iamClient) GetWithVersion(ctx context.Context, resource string, request call.UserProject(c.userProject) } var rp *raw.Policy - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { rp, err = call.Context(ctx).Do() return err - }) + }, c.retry, true) if err != nil { return nil, err } @@ -72,10 +74,11 @@ func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) ( if c.userProject != "" { call.UserProject(c.userProject) } - return runWithRetry(ctx, func() error { + isIdempotent := len(p.Etag) > 0 + return run(ctx, func() error { _, err := call.Context(ctx).Do() return err - }) + }, c.retry, isIdempotent) } func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (permissions []string, err error) { @@ -88,10 +91,10 @@ func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ( call.UserProject(c.userProject) } var res *raw.TestIamPermissionsResponse - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { res, err = call.Context(ctx).Do() return err - }) + }, c.retry, true) if err != nil { return nil, err } diff --git a/vendor/cloud.google.com/go/storage/internal/apiv2/doc.go b/vendor/cloud.google.com/go/storage/internal/apiv2/doc.go index 5bbc94e5e1..f5ed5c8299 100644 --- a/vendor/cloud.google.com/go/storage/internal/apiv2/doc.go +++ b/vendor/cloud.google.com/go/storage/internal/apiv2/doc.go @@ -1,4 +1,4 @@ -// Copyright 2021 Google LLC +// Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -39,6 +39,22 @@ // // The following is an example of making an API call with the newly created client. // +// ctx := context.Background() +// c, err := storage.NewClient(ctx) +// if err != nil { +// // TODO: Handle error. +// } +// defer c.Close() +// +// req := &storagepb.DeleteBucketRequest{ +// // TODO: Fill request struct fields. +// // See https://pkg.go.dev/google.golang.org/genproto/googleapis/storage/v2#DeleteBucketRequest. +// } +// err = c.DeleteBucket(ctx, req) +// if err != nil { +// // TODO: Handle error. +// } +// // Use of Context // // The ctx passed to NewClient is used for authentication requests and @@ -68,7 +84,7 @@ import ( type clientHookParams struct{} type clientHook func(context.Context, clientHookParams) ([]option.ClientOption, error) -const versionClient = "20211015" +const versionClient = "20220114" func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context { out, _ := metadata.FromOutgoingContext(ctx) diff --git a/vendor/cloud.google.com/go/storage/internal/apiv2/gapic_metadata.json b/vendor/cloud.google.com/go/storage/internal/apiv2/gapic_metadata.json index 5f7961b152..355215ed41 100644 --- a/vendor/cloud.google.com/go/storage/internal/apiv2/gapic_metadata.json +++ b/vendor/cloud.google.com/go/storage/internal/apiv2/gapic_metadata.json @@ -10,6 +10,101 @@ "grpc": { "libraryClient": "Client", "rpcs": { + "ComposeObject": { + "methods": [ + "ComposeObject" + ] + }, + "CreateBucket": { + "methods": [ + "CreateBucket" + ] + }, + "CreateHmacKey": { + "methods": [ + "CreateHmacKey" + ] + }, + "CreateNotification": { + "methods": [ + "CreateNotification" + ] + }, + "DeleteBucket": { + "methods": [ + "DeleteBucket" + ] + }, + "DeleteHmacKey": { + "methods": [ + "DeleteHmacKey" + ] + }, + "DeleteNotification": { + "methods": [ + "DeleteNotification" + ] + }, + "DeleteObject": { + "methods": [ + "DeleteObject" + ] + }, + "GetBucket": { + "methods": [ + "GetBucket" + ] + }, + "GetHmacKey": { + "methods": [ + "GetHmacKey" + ] + }, + "GetIamPolicy": { + "methods": [ + "GetIamPolicy" + ] + }, + "GetNotification": { + "methods": [ + "GetNotification" + ] + }, + "GetObject": { + "methods": [ + "GetObject" + ] + }, + "GetServiceAccount": { + "methods": [ + "GetServiceAccount" + ] + }, + "ListBuckets": { + "methods": [ + "ListBuckets" + ] + }, + "ListHmacKeys": { + "methods": [ + "ListHmacKeys" + ] + }, + "ListNotifications": { + "methods": [ + "ListNotifications" + ] + }, + "ListObjects": { + "methods": [ + "ListObjects" + ] + }, + "LockBucketRetentionPolicy": { + "methods": [ + "LockBucketRetentionPolicy" + ] + }, "QueryWriteStatus": { "methods": [ "QueryWriteStatus" @@ -20,11 +115,41 @@ "ReadObject" ] }, + "RewriteObject": { + "methods": [ + "RewriteObject" + ] + }, + "SetIamPolicy": { + "methods": [ + "SetIamPolicy" + ] + }, "StartResumableWrite": { "methods": [ "StartResumableWrite" ] }, + "TestIamPermissions": { + "methods": [ + "TestIamPermissions" + ] + }, + "UpdateBucket": { + "methods": [ + "UpdateBucket" + ] + }, + "UpdateHmacKey": { + "methods": [ + "UpdateHmacKey" + ] + }, + "UpdateObject": { + "methods": [ + "UpdateObject" + ] + }, "WriteObject": { "methods": [ "WriteObject" diff --git a/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go b/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go index 261ae13d05..8507a0c0f3 100644 --- a/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go +++ b/vendor/cloud.google.com/go/storage/internal/apiv2/storage_client.go @@ -1,4 +1,4 @@ -// Copyright 2021 Google LLC +// Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,23 +22,51 @@ import ( "time" gax "github.com/googleapis/gax-go/v2" + "google.golang.org/api/iterator" "google.golang.org/api/option" "google.golang.org/api/option/internaloption" gtransport "google.golang.org/api/transport/grpc" + iampb "google.golang.org/genproto/googleapis/iam/v1" storagepb "google.golang.org/genproto/googleapis/storage/v2" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" ) var newClientHook clientHook // CallOptions contains the retry settings for each method of Client. type CallOptions struct { - ReadObject []gax.CallOption - WriteObject []gax.CallOption - StartResumableWrite []gax.CallOption - QueryWriteStatus []gax.CallOption + DeleteBucket []gax.CallOption + GetBucket []gax.CallOption + CreateBucket []gax.CallOption + ListBuckets []gax.CallOption + LockBucketRetentionPolicy []gax.CallOption + GetIamPolicy []gax.CallOption + SetIamPolicy []gax.CallOption + TestIamPermissions []gax.CallOption + UpdateBucket []gax.CallOption + DeleteNotification []gax.CallOption + GetNotification []gax.CallOption + CreateNotification []gax.CallOption + ListNotifications []gax.CallOption + ComposeObject []gax.CallOption + DeleteObject []gax.CallOption + GetObject []gax.CallOption + ReadObject []gax.CallOption + UpdateObject []gax.CallOption + WriteObject []gax.CallOption + ListObjects []gax.CallOption + RewriteObject []gax.CallOption + StartResumableWrite []gax.CallOption + QueryWriteStatus []gax.CallOption + GetServiceAccount []gax.CallOption + CreateHmacKey []gax.CallOption + DeleteHmacKey []gax.CallOption + GetHmacKey []gax.CallOption + ListHmacKeys []gax.CallOption + UpdateHmacKey []gax.CallOption } func defaultGRPCClientOptions() []option.ClientOption { @@ -48,7 +76,6 @@ func defaultGRPCClientOptions() []option.ClientOption { internaloption.WithDefaultAudience("https://storage.googleapis.com/"), internaloption.WithDefaultScopes(DefaultAuthScopes()...), internaloption.EnableJwtWithScope(), - option.WithGRPCDialOption(grpc.WithDisableServiceConfig()), option.WithGRPCDialOption(grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(math.MaxInt32))), } @@ -56,6 +83,198 @@ func defaultGRPCClientOptions() []option.ClientOption { func defaultCallOptions() *CallOptions { return &CallOptions{ + DeleteBucket: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + GetBucket: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + CreateBucket: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + ListBuckets: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + LockBucketRetentionPolicy: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + GetIamPolicy: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + SetIamPolicy: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + TestIamPermissions: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + UpdateBucket: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + DeleteNotification: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + GetNotification: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + CreateNotification: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + ListNotifications: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + ComposeObject: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + DeleteObject: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + GetObject: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, ReadObject: []gax.CallOption{ gax.WithRetry(func() gax.Retryer { return gax.OnCodes([]codes.Code{ @@ -68,6 +287,18 @@ func defaultCallOptions() *CallOptions { }) }), }, + UpdateObject: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, WriteObject: []gax.CallOption{ gax.WithRetry(func() gax.Retryer { return gax.OnCodes([]codes.Code{ @@ -80,6 +311,30 @@ func defaultCallOptions() *CallOptions { }) }), }, + ListObjects: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + RewriteObject: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, StartResumableWrite: []gax.CallOption{ gax.WithRetry(func() gax.Retryer { return gax.OnCodes([]codes.Code{ @@ -104,6 +359,78 @@ func defaultCallOptions() *CallOptions { }) }), }, + GetServiceAccount: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + CreateHmacKey: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + DeleteHmacKey: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + GetHmacKey: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + ListHmacKeys: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, + UpdateHmacKey: []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnCodes([]codes.Code{ + codes.DeadlineExceeded, + codes.Unavailable, + }, gax.Backoff{ + Initial: 1000 * time.Millisecond, + Max: 60000 * time.Millisecond, + Multiplier: 2.00, + }) + }), + }, } } @@ -112,16 +439,62 @@ type internalClient interface { Close() error setGoogleClientInfo(...string) Connection() *grpc.ClientConn + DeleteBucket(context.Context, *storagepb.DeleteBucketRequest, ...gax.CallOption) error + GetBucket(context.Context, *storagepb.GetBucketRequest, ...gax.CallOption) (*storagepb.Bucket, error) + CreateBucket(context.Context, *storagepb.CreateBucketRequest, ...gax.CallOption) (*storagepb.Bucket, error) + ListBuckets(context.Context, *storagepb.ListBucketsRequest, ...gax.CallOption) *BucketIterator + LockBucketRetentionPolicy(context.Context, *storagepb.LockBucketRetentionPolicyRequest, ...gax.CallOption) (*storagepb.Bucket, error) + GetIamPolicy(context.Context, *iampb.GetIamPolicyRequest, ...gax.CallOption) (*iampb.Policy, error) + SetIamPolicy(context.Context, *iampb.SetIamPolicyRequest, ...gax.CallOption) (*iampb.Policy, error) + TestIamPermissions(context.Context, *iampb.TestIamPermissionsRequest, ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) + UpdateBucket(context.Context, *storagepb.UpdateBucketRequest, ...gax.CallOption) (*storagepb.Bucket, error) + DeleteNotification(context.Context, *storagepb.DeleteNotificationRequest, ...gax.CallOption) error + GetNotification(context.Context, *storagepb.GetNotificationRequest, ...gax.CallOption) (*storagepb.Notification, error) + CreateNotification(context.Context, *storagepb.CreateNotificationRequest, ...gax.CallOption) (*storagepb.Notification, error) + ListNotifications(context.Context, *storagepb.ListNotificationsRequest, ...gax.CallOption) *NotificationIterator + ComposeObject(context.Context, *storagepb.ComposeObjectRequest, ...gax.CallOption) (*storagepb.Object, error) + DeleteObject(context.Context, *storagepb.DeleteObjectRequest, ...gax.CallOption) error + GetObject(context.Context, *storagepb.GetObjectRequest, ...gax.CallOption) (*storagepb.Object, error) ReadObject(context.Context, *storagepb.ReadObjectRequest, ...gax.CallOption) (storagepb.Storage_ReadObjectClient, error) + UpdateObject(context.Context, *storagepb.UpdateObjectRequest, ...gax.CallOption) (*storagepb.Object, error) WriteObject(context.Context, ...gax.CallOption) (storagepb.Storage_WriteObjectClient, error) + ListObjects(context.Context, *storagepb.ListObjectsRequest, ...gax.CallOption) *ObjectIterator + RewriteObject(context.Context, *storagepb.RewriteObjectRequest, ...gax.CallOption) (*storagepb.RewriteResponse, error) StartResumableWrite(context.Context, *storagepb.StartResumableWriteRequest, ...gax.CallOption) (*storagepb.StartResumableWriteResponse, error) QueryWriteStatus(context.Context, *storagepb.QueryWriteStatusRequest, ...gax.CallOption) (*storagepb.QueryWriteStatusResponse, error) + GetServiceAccount(context.Context, *storagepb.GetServiceAccountRequest, ...gax.CallOption) (*storagepb.ServiceAccount, error) + CreateHmacKey(context.Context, *storagepb.CreateHmacKeyRequest, ...gax.CallOption) (*storagepb.CreateHmacKeyResponse, error) + DeleteHmacKey(context.Context, *storagepb.DeleteHmacKeyRequest, ...gax.CallOption) error + GetHmacKey(context.Context, *storagepb.GetHmacKeyRequest, ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) + ListHmacKeys(context.Context, *storagepb.ListHmacKeysRequest, ...gax.CallOption) *HmacKeyMetadataIterator + UpdateHmacKey(context.Context, *storagepb.UpdateHmacKeyRequest, ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) } // Client is a client for interacting with Cloud Storage API. // Methods, except Close, may be called concurrently. However, fields must not be modified concurrently with method calls. // -// Manages Google Cloud Storage resources. +// API Overview and Naming SyntaxThe Cloud Storage gRPC API allows applications to read and write data through +// the abstractions of buckets and objects. For a description of these +// abstractions please see https://cloud.google.com/storage/docs (at https://cloud.google.com/storage/docs). +// +// Resources are named as follows: +// +// Projects are referred to as they are defined by the Resource Manager API, +// using strings like projects/123456 or projects/my-string-id. +// +// Buckets are named using string names of the form: +// projects/{project}/buckets/{bucket} +// For globally unique buckets, _ may be substituted for the project. +// +// Objects are uniquely identified by their name along with the name of the +// bucket they belong to, as separate strings in this API. For example: +// +// ReadObjectRequest { +// bucket: ‘projects/_/buckets/my-bucket’ +// object: ‘my-object’ +// } +// Note that object names can contain / characters, which are treated as +// any other character (no special directory semantics). type Client struct { // The internal transport-dependent client. internalClient internalClient @@ -138,18 +511,105 @@ func (c *Client) Close() error { return c.internalClient.Close() } -// setGoogleClientInfo sets the name and version of the application in -// the `x-goog-api-client` header passed on each request. Intended for -// use by Google-written clients. -func (c *Client) setGoogleClientInfo(keyval ...string) { - c.internalClient.setGoogleClientInfo(keyval...) +// setGoogleClientInfo sets the name and version of the application in +// the `x-goog-api-client` header passed on each request. Intended for +// use by Google-written clients. +func (c *Client) setGoogleClientInfo(keyval ...string) { + c.internalClient.setGoogleClientInfo(keyval...) +} + +// Connection returns a connection to the API service. +// +// Deprecated. +func (c *Client) Connection() *grpc.ClientConn { + return c.internalClient.Connection() +} + +// DeleteBucket permanently deletes an empty bucket. +func (c *Client) DeleteBucket(ctx context.Context, req *storagepb.DeleteBucketRequest, opts ...gax.CallOption) error { + return c.internalClient.DeleteBucket(ctx, req, opts...) +} + +// GetBucket returns metadata for the specified bucket. +func (c *Client) GetBucket(ctx context.Context, req *storagepb.GetBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + return c.internalClient.GetBucket(ctx, req, opts...) +} + +// CreateBucket creates a new bucket. +func (c *Client) CreateBucket(ctx context.Context, req *storagepb.CreateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + return c.internalClient.CreateBucket(ctx, req, opts...) +} + +// ListBuckets retrieves a list of buckets for a given project. +func (c *Client) ListBuckets(ctx context.Context, req *storagepb.ListBucketsRequest, opts ...gax.CallOption) *BucketIterator { + return c.internalClient.ListBuckets(ctx, req, opts...) +} + +// LockBucketRetentionPolicy locks retention policy on a bucket. +func (c *Client) LockBucketRetentionPolicy(ctx context.Context, req *storagepb.LockBucketRetentionPolicyRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + return c.internalClient.LockBucketRetentionPolicy(ctx, req, opts...) +} + +// GetIamPolicy gets the IAM policy for a specified bucket. +func (c *Client) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { + return c.internalClient.GetIamPolicy(ctx, req, opts...) +} + +// SetIamPolicy updates an IAM policy for the specified bucket. +func (c *Client) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { + return c.internalClient.SetIamPolicy(ctx, req, opts...) +} + +// TestIamPermissions tests a set of permissions on the given bucket to see which, if +// any, are held by the caller. +func (c *Client) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) { + return c.internalClient.TestIamPermissions(ctx, req, opts...) +} + +// UpdateBucket updates a bucket. Equivalent to JSON API’s storage.buckets.patch method. +func (c *Client) UpdateBucket(ctx context.Context, req *storagepb.UpdateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + return c.internalClient.UpdateBucket(ctx, req, opts...) +} + +// DeleteNotification permanently deletes a notification subscription. +func (c *Client) DeleteNotification(ctx context.Context, req *storagepb.DeleteNotificationRequest, opts ...gax.CallOption) error { + return c.internalClient.DeleteNotification(ctx, req, opts...) +} + +// GetNotification view a notification config. +func (c *Client) GetNotification(ctx context.Context, req *storagepb.GetNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) { + return c.internalClient.GetNotification(ctx, req, opts...) +} + +// CreateNotification creates a notification subscription for a given bucket. +// These notifications, when triggered, publish messages to the specified +// Pub/Sub topics. +// See https://cloud.google.com/storage/docs/pubsub-notifications (at https://cloud.google.com/storage/docs/pubsub-notifications). +func (c *Client) CreateNotification(ctx context.Context, req *storagepb.CreateNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) { + return c.internalClient.CreateNotification(ctx, req, opts...) +} + +// ListNotifications retrieves a list of notification subscriptions for a given bucket. +func (c *Client) ListNotifications(ctx context.Context, req *storagepb.ListNotificationsRequest, opts ...gax.CallOption) *NotificationIterator { + return c.internalClient.ListNotifications(ctx, req, opts...) +} + +// ComposeObject concatenates a list of existing objects into a new object in the same +// bucket. +func (c *Client) ComposeObject(ctx context.Context, req *storagepb.ComposeObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + return c.internalClient.ComposeObject(ctx, req, opts...) } -// Connection returns a connection to the API service. -// -// Deprecated. -func (c *Client) Connection() *grpc.ClientConn { - return c.internalClient.Connection() +// DeleteObject deletes an object and its metadata. Deletions are permanent if versioning +// is not enabled for the bucket, or if the generation parameter +// is used. +func (c *Client) DeleteObject(ctx context.Context, req *storagepb.DeleteObjectRequest, opts ...gax.CallOption) error { + return c.internalClient.DeleteObject(ctx, req, opts...) +} + +// GetObject retrieves an object’s metadata. +func (c *Client) GetObject(ctx context.Context, req *storagepb.GetObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + return c.internalClient.GetObject(ctx, req, opts...) } // ReadObject reads an object’s data. @@ -157,6 +617,12 @@ func (c *Client) ReadObject(ctx context.Context, req *storagepb.ReadObjectReques return c.internalClient.ReadObject(ctx, req, opts...) } +// UpdateObject updates an object’s metadata. +// Equivalent to JSON API’s storage.objects.patch. +func (c *Client) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + return c.internalClient.UpdateObject(ctx, req, opts...) +} + // WriteObject stores a new object and metadata. // // An object can be written either in a single message stream or in a @@ -185,6 +651,17 @@ func (c *Client) WriteObject(ctx context.Context, opts ...gax.CallOption) (stora return c.internalClient.WriteObject(ctx, opts...) } +// ListObjects retrieves a list of objects matching the criteria. +func (c *Client) ListObjects(ctx context.Context, req *storagepb.ListObjectsRequest, opts ...gax.CallOption) *ObjectIterator { + return c.internalClient.ListObjects(ctx, req, opts...) +} + +// RewriteObject rewrites a source object to a destination object. Optionally overrides +// metadata. +func (c *Client) RewriteObject(ctx context.Context, req *storagepb.RewriteObjectRequest, opts ...gax.CallOption) (*storagepb.RewriteResponse, error) { + return c.internalClient.RewriteObject(ctx, req, opts...) +} + // StartResumableWrite starts a resumable write. How long the write operation remains valid, and // what happens when the write operation becomes invalid, are // service-dependent. @@ -209,6 +686,36 @@ func (c *Client) QueryWriteStatus(ctx context.Context, req *storagepb.QueryWrite return c.internalClient.QueryWriteStatus(ctx, req, opts...) } +// GetServiceAccount retrieves the name of a project’s Google Cloud Storage service account. +func (c *Client) GetServiceAccount(ctx context.Context, req *storagepb.GetServiceAccountRequest, opts ...gax.CallOption) (*storagepb.ServiceAccount, error) { + return c.internalClient.GetServiceAccount(ctx, req, opts...) +} + +// CreateHmacKey creates a new HMAC key for the given service account. +func (c *Client) CreateHmacKey(ctx context.Context, req *storagepb.CreateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.CreateHmacKeyResponse, error) { + return c.internalClient.CreateHmacKey(ctx, req, opts...) +} + +// DeleteHmacKey deletes a given HMAC key. Key must be in an INACTIVE state. +func (c *Client) DeleteHmacKey(ctx context.Context, req *storagepb.DeleteHmacKeyRequest, opts ...gax.CallOption) error { + return c.internalClient.DeleteHmacKey(ctx, req, opts...) +} + +// GetHmacKey gets an existing HMAC key metadata for the given id. +func (c *Client) GetHmacKey(ctx context.Context, req *storagepb.GetHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) { + return c.internalClient.GetHmacKey(ctx, req, opts...) +} + +// ListHmacKeys lists HMAC keys under a given project with the additional filters provided. +func (c *Client) ListHmacKeys(ctx context.Context, req *storagepb.ListHmacKeysRequest, opts ...gax.CallOption) *HmacKeyMetadataIterator { + return c.internalClient.ListHmacKeys(ctx, req, opts...) +} + +// UpdateHmacKey updates a given HMAC key state between ACTIVE and INACTIVE. +func (c *Client) UpdateHmacKey(ctx context.Context, req *storagepb.UpdateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) { + return c.internalClient.UpdateHmacKey(ctx, req, opts...) +} + // gRPCClient is a client for interacting with Cloud Storage API over gRPC transport. // // Methods, except Close, may be called concurrently. However, fields must not be modified concurrently with method calls. @@ -232,7 +739,28 @@ type gRPCClient struct { // NewClient creates a new storage client based on gRPC. // The returned client must be Closed when it is done being used to clean up its underlying connections. // -// Manages Google Cloud Storage resources. +// API Overview and Naming SyntaxThe Cloud Storage gRPC API allows applications to read and write data through +// the abstractions of buckets and objects. For a description of these +// abstractions please see https://cloud.google.com/storage/docs (at https://cloud.google.com/storage/docs). +// +// Resources are named as follows: +// +// Projects are referred to as they are defined by the Resource Manager API, +// using strings like projects/123456 or projects/my-string-id. +// +// Buckets are named using string names of the form: +// projects/{project}/buckets/{bucket} +// For globally unique buckets, _ may be substituted for the project. +// +// Objects are uniquely identified by their name along with the name of the +// bucket they belong to, as separate strings in this API. For example: +// +// ReadObjectRequest { +// bucket: ‘projects/_/buckets/my-bucket’ +// object: ‘my-object’ +// } +// Note that object names can contain / characters, which are treated as +// any other character (no special directory semantics). func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { clientOpts := defaultGRPCClientOptions() if newClientHook != nil { @@ -254,39 +782,393 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error } client := Client{CallOptions: defaultCallOptions()} - c := &gRPCClient{ - connPool: connPool, - disableDeadlines: disableDeadlines, - client: storagepb.NewStorageClient(connPool), - CallOptions: &client.CallOptions, + c := &gRPCClient{ + connPool: connPool, + disableDeadlines: disableDeadlines, + client: storagepb.NewStorageClient(connPool), + CallOptions: &client.CallOptions, + } + c.setGoogleClientInfo() + + client.internalClient = c + + return &client, nil +} + +// Connection returns a connection to the API service. +// +// Deprecated. +func (c *gRPCClient) Connection() *grpc.ClientConn { + return c.connPool.Conn() +} + +// setGoogleClientInfo sets the name and version of the application in +// the `x-goog-api-client` header passed on each request. Intended for +// use by Google-written clients. +func (c *gRPCClient) setGoogleClientInfo(keyval ...string) { + kv := append([]string{"gl-go", versionGo()}, keyval...) + kv = append(kv, "gapic", versionClient, "gax", gax.Version, "grpc", grpc.Version) + c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...)) +} + +// Close closes the connection to the API service. The user should invoke this when +// the client is no longer required. +func (c *gRPCClient) Close() error { + return c.connPool.Close() +} + +func (c *gRPCClient) DeleteBucket(ctx context.Context, req *storagepb.DeleteBucketRequest, opts ...gax.CallOption) error { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).DeleteBucket[0:len((*c.CallOptions).DeleteBucket):len((*c.CallOptions).DeleteBucket)], opts...) + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + _, err = c.client.DeleteBucket(ctx, req, settings.GRPC...) + return err + }, opts...) + return err +} + +func (c *gRPCClient) GetBucket(ctx context.Context, req *storagepb.GetBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetBucket[0:len((*c.CallOptions).GetBucket):len((*c.CallOptions).GetBucket)], opts...) + var resp *storagepb.Bucket + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetBucket(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) CreateBucket(ctx context.Context, req *storagepb.CreateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).CreateBucket[0:len((*c.CallOptions).CreateBucket):len((*c.CallOptions).CreateBucket)], opts...) + var resp *storagepb.Bucket + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.CreateBucket(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) ListBuckets(ctx context.Context, req *storagepb.ListBucketsRequest, opts ...gax.CallOption) *BucketIterator { + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).ListBuckets[0:len((*c.CallOptions).ListBuckets):len((*c.CallOptions).ListBuckets)], opts...) + it := &BucketIterator{} + req = proto.Clone(req).(*storagepb.ListBucketsRequest) + it.InternalFetch = func(pageSize int, pageToken string) ([]*storagepb.Bucket, string, error) { + resp := &storagepb.ListBucketsResponse{} + if pageToken != "" { + req.PageToken = pageToken + } + if pageSize > math.MaxInt32 { + req.PageSize = math.MaxInt32 + } else if pageSize != 0 { + req.PageSize = int32(pageSize) + } + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.ListBuckets(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, "", err + } + + it.Response = resp + return resp.GetBuckets(), resp.GetNextPageToken(), nil + } + fetch := func(pageSize int, pageToken string) (string, error) { + items, nextPageToken, err := it.InternalFetch(pageSize, pageToken) + if err != nil { + return "", err + } + it.items = append(it.items, items...) + return nextPageToken, nil + } + + it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf) + it.pageInfo.MaxSize = int(req.GetPageSize()) + it.pageInfo.Token = req.GetPageToken() + + return it +} + +func (c *gRPCClient) LockBucketRetentionPolicy(ctx context.Context, req *storagepb.LockBucketRetentionPolicyRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).LockBucketRetentionPolicy[0:len((*c.CallOptions).LockBucketRetentionPolicy):len((*c.CallOptions).LockBucketRetentionPolicy)], opts...) + var resp *storagepb.Bucket + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.LockBucketRetentionPolicy(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) GetIamPolicy(ctx context.Context, req *iampb.GetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetIamPolicy[0:len((*c.CallOptions).GetIamPolicy):len((*c.CallOptions).GetIamPolicy)], opts...) + var resp *iampb.Policy + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetIamPolicy(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) SetIamPolicy(ctx context.Context, req *iampb.SetIamPolicyRequest, opts ...gax.CallOption) (*iampb.Policy, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).SetIamPolicy[0:len((*c.CallOptions).SetIamPolicy):len((*c.CallOptions).SetIamPolicy)], opts...) + var resp *iampb.Policy + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.SetIamPolicy(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) TestIamPermissions(ctx context.Context, req *iampb.TestIamPermissionsRequest, opts ...gax.CallOption) (*iampb.TestIamPermissionsResponse, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).TestIamPermissions[0:len((*c.CallOptions).TestIamPermissions):len((*c.CallOptions).TestIamPermissions)], opts...) + var resp *iampb.TestIamPermissionsResponse + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.TestIamPermissions(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) UpdateBucket(ctx context.Context, req *storagepb.UpdateBucketRequest, opts ...gax.CallOption) (*storagepb.Bucket, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).UpdateBucket[0:len((*c.CallOptions).UpdateBucket):len((*c.CallOptions).UpdateBucket)], opts...) + var resp *storagepb.Bucket + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.UpdateBucket(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) DeleteNotification(ctx context.Context, req *storagepb.DeleteNotificationRequest, opts ...gax.CallOption) error { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).DeleteNotification[0:len((*c.CallOptions).DeleteNotification):len((*c.CallOptions).DeleteNotification)], opts...) + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + _, err = c.client.DeleteNotification(ctx, req, settings.GRPC...) + return err + }, opts...) + return err +} + +func (c *gRPCClient) GetNotification(ctx context.Context, req *storagepb.GetNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetNotification[0:len((*c.CallOptions).GetNotification):len((*c.CallOptions).GetNotification)], opts...) + var resp *storagepb.Notification + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetNotification(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) CreateNotification(ctx context.Context, req *storagepb.CreateNotificationRequest, opts ...gax.CallOption) (*storagepb.Notification, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).CreateNotification[0:len((*c.CallOptions).CreateNotification):len((*c.CallOptions).CreateNotification)], opts...) + var resp *storagepb.Notification + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.CreateNotification(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) ListNotifications(ctx context.Context, req *storagepb.ListNotificationsRequest, opts ...gax.CallOption) *NotificationIterator { + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).ListNotifications[0:len((*c.CallOptions).ListNotifications):len((*c.CallOptions).ListNotifications)], opts...) + it := &NotificationIterator{} + req = proto.Clone(req).(*storagepb.ListNotificationsRequest) + it.InternalFetch = func(pageSize int, pageToken string) ([]*storagepb.Notification, string, error) { + resp := &storagepb.ListNotificationsResponse{} + if pageToken != "" { + req.PageToken = pageToken + } + if pageSize > math.MaxInt32 { + req.PageSize = math.MaxInt32 + } else if pageSize != 0 { + req.PageSize = int32(pageSize) + } + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.ListNotifications(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, "", err + } + + it.Response = resp + return resp.GetNotifications(), resp.GetNextPageToken(), nil + } + fetch := func(pageSize int, pageToken string) (string, error) { + items, nextPageToken, err := it.InternalFetch(pageSize, pageToken) + if err != nil { + return "", err + } + it.items = append(it.items, items...) + return nextPageToken, nil } - c.setGoogleClientInfo() - client.internalClient = c + it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf) + it.pageInfo.MaxSize = int(req.GetPageSize()) + it.pageInfo.Token = req.GetPageToken() - return &client, nil + return it } -// Connection returns a connection to the API service. -// -// Deprecated. -func (c *gRPCClient) Connection() *grpc.ClientConn { - return c.connPool.Conn() +func (c *gRPCClient) ComposeObject(ctx context.Context, req *storagepb.ComposeObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).ComposeObject[0:len((*c.CallOptions).ComposeObject):len((*c.CallOptions).ComposeObject)], opts...) + var resp *storagepb.Object + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.ComposeObject(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil } -// setGoogleClientInfo sets the name and version of the application in -// the `x-goog-api-client` header passed on each request. Intended for -// use by Google-written clients. -func (c *gRPCClient) setGoogleClientInfo(keyval ...string) { - kv := append([]string{"gl-go", versionGo()}, keyval...) - kv = append(kv, "gapic", versionClient, "gax", gax.Version, "grpc", grpc.Version) - c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...)) +func (c *gRPCClient) DeleteObject(ctx context.Context, req *storagepb.DeleteObjectRequest, opts ...gax.CallOption) error { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).DeleteObject[0:len((*c.CallOptions).DeleteObject):len((*c.CallOptions).DeleteObject)], opts...) + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + _, err = c.client.DeleteObject(ctx, req, settings.GRPC...) + return err + }, opts...) + return err } -// Close closes the connection to the API service. The user should invoke this when -// the client is no longer required. -func (c *gRPCClient) Close() error { - return c.connPool.Close() +func (c *gRPCClient) GetObject(ctx context.Context, req *storagepb.GetObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetObject[0:len((*c.CallOptions).GetObject):len((*c.CallOptions).GetObject)], opts...) + var resp *storagepb.Object + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetObject(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil } func (c *gRPCClient) ReadObject(ctx context.Context, req *storagepb.ReadObjectRequest, opts ...gax.CallOption) (storagepb.Storage_ReadObjectClient, error) { @@ -303,6 +1185,26 @@ func (c *gRPCClient) ReadObject(ctx context.Context, req *storagepb.ReadObjectRe return resp, nil } +func (c *gRPCClient) UpdateObject(ctx context.Context, req *storagepb.UpdateObjectRequest, opts ...gax.CallOption) (*storagepb.Object, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).UpdateObject[0:len((*c.CallOptions).UpdateObject):len((*c.CallOptions).UpdateObject)], opts...) + var resp *storagepb.Object + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.UpdateObject(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + func (c *gRPCClient) WriteObject(ctx context.Context, opts ...gax.CallOption) (storagepb.Storage_WriteObjectClient, error) { ctx = insertMetadata(ctx, c.xGoogMetadata) var resp storagepb.Storage_WriteObjectClient @@ -318,6 +1220,69 @@ func (c *gRPCClient) WriteObject(ctx context.Context, opts ...gax.CallOption) (s return resp, nil } +func (c *gRPCClient) ListObjects(ctx context.Context, req *storagepb.ListObjectsRequest, opts ...gax.CallOption) *ObjectIterator { + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).ListObjects[0:len((*c.CallOptions).ListObjects):len((*c.CallOptions).ListObjects)], opts...) + it := &ObjectIterator{} + req = proto.Clone(req).(*storagepb.ListObjectsRequest) + it.InternalFetch = func(pageSize int, pageToken string) ([]*storagepb.Object, string, error) { + resp := &storagepb.ListObjectsResponse{} + if pageToken != "" { + req.PageToken = pageToken + } + if pageSize > math.MaxInt32 { + req.PageSize = math.MaxInt32 + } else if pageSize != 0 { + req.PageSize = int32(pageSize) + } + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.ListObjects(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, "", err + } + + it.Response = resp + return resp.GetObjects(), resp.GetNextPageToken(), nil + } + fetch := func(pageSize int, pageToken string) (string, error) { + items, nextPageToken, err := it.InternalFetch(pageSize, pageToken) + if err != nil { + return "", err + } + it.items = append(it.items, items...) + return nextPageToken, nil + } + + it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf) + it.pageInfo.MaxSize = int(req.GetPageSize()) + it.pageInfo.Token = req.GetPageToken() + + return it +} + +func (c *gRPCClient) RewriteObject(ctx context.Context, req *storagepb.RewriteObjectRequest, opts ...gax.CallOption) (*storagepb.RewriteResponse, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).RewriteObject[0:len((*c.CallOptions).RewriteObject):len((*c.CallOptions).RewriteObject)], opts...) + var resp *storagepb.RewriteResponse + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.RewriteObject(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + func (c *gRPCClient) StartResumableWrite(ctx context.Context, req *storagepb.StartResumableWriteRequest, opts ...gax.CallOption) (*storagepb.StartResumableWriteResponse, error) { if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) @@ -357,3 +1322,330 @@ func (c *gRPCClient) QueryWriteStatus(ctx context.Context, req *storagepb.QueryW } return resp, nil } + +func (c *gRPCClient) GetServiceAccount(ctx context.Context, req *storagepb.GetServiceAccountRequest, opts ...gax.CallOption) (*storagepb.ServiceAccount, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetServiceAccount[0:len((*c.CallOptions).GetServiceAccount):len((*c.CallOptions).GetServiceAccount)], opts...) + var resp *storagepb.ServiceAccount + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetServiceAccount(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) CreateHmacKey(ctx context.Context, req *storagepb.CreateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.CreateHmacKeyResponse, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).CreateHmacKey[0:len((*c.CallOptions).CreateHmacKey):len((*c.CallOptions).CreateHmacKey)], opts...) + var resp *storagepb.CreateHmacKeyResponse + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.CreateHmacKey(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) DeleteHmacKey(ctx context.Context, req *storagepb.DeleteHmacKeyRequest, opts ...gax.CallOption) error { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).DeleteHmacKey[0:len((*c.CallOptions).DeleteHmacKey):len((*c.CallOptions).DeleteHmacKey)], opts...) + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + _, err = c.client.DeleteHmacKey(ctx, req, settings.GRPC...) + return err + }, opts...) + return err +} + +func (c *gRPCClient) GetHmacKey(ctx context.Context, req *storagepb.GetHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).GetHmacKey[0:len((*c.CallOptions).GetHmacKey):len((*c.CallOptions).GetHmacKey)], opts...) + var resp *storagepb.HmacKeyMetadata + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.GetHmacKey(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *gRPCClient) ListHmacKeys(ctx context.Context, req *storagepb.ListHmacKeysRequest, opts ...gax.CallOption) *HmacKeyMetadataIterator { + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).ListHmacKeys[0:len((*c.CallOptions).ListHmacKeys):len((*c.CallOptions).ListHmacKeys)], opts...) + it := &HmacKeyMetadataIterator{} + req = proto.Clone(req).(*storagepb.ListHmacKeysRequest) + it.InternalFetch = func(pageSize int, pageToken string) ([]*storagepb.HmacKeyMetadata, string, error) { + resp := &storagepb.ListHmacKeysResponse{} + if pageToken != "" { + req.PageToken = pageToken + } + if pageSize > math.MaxInt32 { + req.PageSize = math.MaxInt32 + } else if pageSize != 0 { + req.PageSize = int32(pageSize) + } + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.ListHmacKeys(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, "", err + } + + it.Response = resp + return resp.GetHmacKeys(), resp.GetNextPageToken(), nil + } + fetch := func(pageSize int, pageToken string) (string, error) { + items, nextPageToken, err := it.InternalFetch(pageSize, pageToken) + if err != nil { + return "", err + } + it.items = append(it.items, items...) + return nextPageToken, nil + } + + it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, it.bufLen, it.takeBuf) + it.pageInfo.MaxSize = int(req.GetPageSize()) + it.pageInfo.Token = req.GetPageToken() + + return it +} + +func (c *gRPCClient) UpdateHmacKey(ctx context.Context, req *storagepb.UpdateHmacKeyRequest, opts ...gax.CallOption) (*storagepb.HmacKeyMetadata, error) { + if _, ok := ctx.Deadline(); !ok && !c.disableDeadlines { + cctx, cancel := context.WithTimeout(ctx, 60000*time.Millisecond) + defer cancel() + ctx = cctx + } + ctx = insertMetadata(ctx, c.xGoogMetadata) + opts = append((*c.CallOptions).UpdateHmacKey[0:len((*c.CallOptions).UpdateHmacKey):len((*c.CallOptions).UpdateHmacKey)], opts...) + var resp *storagepb.HmacKeyMetadata + err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error { + var err error + resp, err = c.client.UpdateHmacKey(ctx, req, settings.GRPC...) + return err + }, opts...) + if err != nil { + return nil, err + } + return resp, nil +} + +// BucketIterator manages a stream of *storagepb.Bucket. +type BucketIterator struct { + items []*storagepb.Bucket + pageInfo *iterator.PageInfo + nextFunc func() error + + // Response is the raw response for the current page. + // It must be cast to the RPC response type. + // Calling Next() or InternalFetch() updates this value. + Response interface{} + + // InternalFetch is for use by the Google Cloud Libraries only. + // It is not part of the stable interface of this package. + // + // InternalFetch returns results from a single call to the underlying RPC. + // The number of results is no greater than pageSize. + // If there are no more results, nextPageToken is empty and err is nil. + InternalFetch func(pageSize int, pageToken string) (results []*storagepb.Bucket, nextPageToken string, err error) +} + +// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. +func (it *BucketIterator) PageInfo() *iterator.PageInfo { + return it.pageInfo +} + +// Next returns the next result. Its second return value is iterator.Done if there are no more +// results. Once Next returns Done, all subsequent calls will return Done. +func (it *BucketIterator) Next() (*storagepb.Bucket, error) { + var item *storagepb.Bucket + if err := it.nextFunc(); err != nil { + return item, err + } + item = it.items[0] + it.items = it.items[1:] + return item, nil +} + +func (it *BucketIterator) bufLen() int { + return len(it.items) +} + +func (it *BucketIterator) takeBuf() interface{} { + b := it.items + it.items = nil + return b +} + +// HmacKeyMetadataIterator manages a stream of *storagepb.HmacKeyMetadata. +type HmacKeyMetadataIterator struct { + items []*storagepb.HmacKeyMetadata + pageInfo *iterator.PageInfo + nextFunc func() error + + // Response is the raw response for the current page. + // It must be cast to the RPC response type. + // Calling Next() or InternalFetch() updates this value. + Response interface{} + + // InternalFetch is for use by the Google Cloud Libraries only. + // It is not part of the stable interface of this package. + // + // InternalFetch returns results from a single call to the underlying RPC. + // The number of results is no greater than pageSize. + // If there are no more results, nextPageToken is empty and err is nil. + InternalFetch func(pageSize int, pageToken string) (results []*storagepb.HmacKeyMetadata, nextPageToken string, err error) +} + +// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. +func (it *HmacKeyMetadataIterator) PageInfo() *iterator.PageInfo { + return it.pageInfo +} + +// Next returns the next result. Its second return value is iterator.Done if there are no more +// results. Once Next returns Done, all subsequent calls will return Done. +func (it *HmacKeyMetadataIterator) Next() (*storagepb.HmacKeyMetadata, error) { + var item *storagepb.HmacKeyMetadata + if err := it.nextFunc(); err != nil { + return item, err + } + item = it.items[0] + it.items = it.items[1:] + return item, nil +} + +func (it *HmacKeyMetadataIterator) bufLen() int { + return len(it.items) +} + +func (it *HmacKeyMetadataIterator) takeBuf() interface{} { + b := it.items + it.items = nil + return b +} + +// NotificationIterator manages a stream of *storagepb.Notification. +type NotificationIterator struct { + items []*storagepb.Notification + pageInfo *iterator.PageInfo + nextFunc func() error + + // Response is the raw response for the current page. + // It must be cast to the RPC response type. + // Calling Next() or InternalFetch() updates this value. + Response interface{} + + // InternalFetch is for use by the Google Cloud Libraries only. + // It is not part of the stable interface of this package. + // + // InternalFetch returns results from a single call to the underlying RPC. + // The number of results is no greater than pageSize. + // If there are no more results, nextPageToken is empty and err is nil. + InternalFetch func(pageSize int, pageToken string) (results []*storagepb.Notification, nextPageToken string, err error) +} + +// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. +func (it *NotificationIterator) PageInfo() *iterator.PageInfo { + return it.pageInfo +} + +// Next returns the next result. Its second return value is iterator.Done if there are no more +// results. Once Next returns Done, all subsequent calls will return Done. +func (it *NotificationIterator) Next() (*storagepb.Notification, error) { + var item *storagepb.Notification + if err := it.nextFunc(); err != nil { + return item, err + } + item = it.items[0] + it.items = it.items[1:] + return item, nil +} + +func (it *NotificationIterator) bufLen() int { + return len(it.items) +} + +func (it *NotificationIterator) takeBuf() interface{} { + b := it.items + it.items = nil + return b +} + +// ObjectIterator manages a stream of *storagepb.Object. +type ObjectIterator struct { + items []*storagepb.Object + pageInfo *iterator.PageInfo + nextFunc func() error + + // Response is the raw response for the current page. + // It must be cast to the RPC response type. + // Calling Next() or InternalFetch() updates this value. + Response interface{} + + // InternalFetch is for use by the Google Cloud Libraries only. + // It is not part of the stable interface of this package. + // + // InternalFetch returns results from a single call to the underlying RPC. + // The number of results is no greater than pageSize. + // If there are no more results, nextPageToken is empty and err is nil. + InternalFetch func(pageSize int, pageToken string) (results []*storagepb.Object, nextPageToken string, err error) +} + +// PageInfo supports pagination. See the google.golang.org/api/iterator package for details. +func (it *ObjectIterator) PageInfo() *iterator.PageInfo { + return it.pageInfo +} + +// Next returns the next result. Its second return value is iterator.Done if there are no more +// results. Once Next returns Done, all subsequent calls will return Done. +func (it *ObjectIterator) Next() (*storagepb.Object, error) { + var item *storagepb.Object + if err := it.nextFunc(); err != nil { + return item, err + } + item = it.items[0] + it.items = it.items[1:] + return item, nil +} + +func (it *ObjectIterator) bufLen() int { + return len(it.items) +} + +func (it *ObjectIterator) takeBuf() interface{} { + b := it.items + it.items = nil + return b +} diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go index 0c44c2db14..aab893d08a 100644 --- a/vendor/cloud.google.com/go/storage/invoke.go +++ b/vendor/cloud.google.com/go/storage/invoke.go @@ -17,42 +17,66 @@ package storage import ( "context" "io" + "net" "net/url" "strings" "cloud.google.com/go/internal" gax "github.com/googleapis/gax-go/v2" + "golang.org/x/xerrors" "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -// runWithRetry calls the function until it returns nil or a non-retryable error, or -// the context is done. -func runWithRetry(ctx context.Context, call func() error) error { - return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) { +var defaultRetry *retryConfig = &retryConfig{} + +// run determines whether a retry is necessary based on the config and +// idempotency information. It then calls the function with or without retries +// as appropriate, using the configured settings. +func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool) error { + if retry == nil { + retry = defaultRetry + } + if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever { + return call() + } + bo := gax.Backoff{} + if retry.backoff != nil { + bo.Multiplier = retry.backoff.Multiplier + bo.Initial = retry.backoff.Initial + bo.Max = retry.backoff.Max + } + var errorFunc func(err error) bool = shouldRetry + if retry.shouldRetry != nil { + errorFunc = retry.shouldRetry + } + return internal.Retry(ctx, bo, func() (stop bool, err error) { err = call() - if err == nil { - return true, nil - } - if shouldRetry(err) { - return false, err - } - return true, err + return !errorFunc(err), err }) } func shouldRetry(err error) bool { - if err == io.ErrUnexpectedEOF { + if err == nil { + return false + } + if xerrors.Is(err, io.ErrUnexpectedEOF) { return true } + switch e := err.(type) { + case *net.OpError: + if strings.Contains(e.Error(), "use of closed network connection") { + // TODO: check against net.ErrClosed (go 1.16+) instead of string + return true + } case *googleapi.Error: - // Retry on 429 and 5xx, according to + // Retry on 408, 429, and 5xx, according to // https://cloud.google.com/storage/docs/exponential-backoff. - return e.Code == 429 || (e.Code >= 500 && e.Code < 600) + return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600) case *url.Error: - // Retry socket-level errors ECONNREFUSED and ENETUNREACH (from syscall). + // Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall). // Unfortunately the error type is unexported, so we resort to string // matching. retriable := []string{"connection refused", "connection reset"} diff --git a/vendor/cloud.google.com/go/storage/notifications.go b/vendor/cloud.google.com/go/storage/notifications.go index 08b3ec3327..333664f92c 100644 --- a/vendor/cloud.google.com/go/storage/notifications.go +++ b/vendor/cloud.google.com/go/storage/notifications.go @@ -137,7 +137,12 @@ func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (re if b.userProject != "" { call.UserProject(b.userProject) } - rn, err := call.Context(ctx).Do() + + var rn *raw.Notification + err = run(ctx, func() error { + rn, err = call.Context(ctx).Do() + return err + }, b.retry, false) if err != nil { return nil, err } @@ -156,10 +161,10 @@ func (b *BucketHandle) Notifications(ctx context.Context) (n map[string]*Notific call.UserProject(b.userProject) } var res *raw.Notifications - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { res, err = call.Context(ctx).Do() return err - }) + }, b.retry, true) if err != nil { return nil, err } @@ -184,7 +189,7 @@ func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err e if b.userProject != "" { call.UserProject(b.userProject) } - return runWithRetry(ctx, func() error { + return run(ctx, func() error { return call.Context(ctx).Do() - }) + }, b.retry, true) } diff --git a/vendor/cloud.google.com/go/storage/post_policy_v4.go b/vendor/cloud.google.com/go/storage/post_policy_v4.go index db9d138384..7e97210101 100644 --- a/vendor/cloud.google.com/go/storage/post_policy_v4.go +++ b/vendor/cloud.google.com/go/storage/post_policy_v4.go @@ -52,22 +52,38 @@ type PostPolicyV4Options struct { // Exactly one of PrivateKey or SignBytes must be non-nil. PrivateKey []byte - // SignBytes is a function for implementing custom signing. For example, if + // SignBytes is a function for implementing custom signing. + // + // Deprecated: Use SignRawBytes. If both SignBytes and SignRawBytes are defined, + // SignBytes will be ignored. + // This SignBytes function expects the bytes it receives to be hashed, while + // SignRawBytes accepts the raw bytes without hashing, allowing more flexibility. + // Add the following to the top of your signing function to hash the bytes + // to use SignRawBytes instead: + // shaSum := sha256.Sum256(bytes) + // bytes = shaSum[:] + // + SignBytes func(hashBytes []byte) (signature []byte, err error) + + // SignRawBytes is a function for implementing custom signing. For example, if // your application is running on Google App Engine, you can use // appengine's internal signing function: - // ctx := appengine.NewContext(request) - // acc, _ := appengine.ServiceAccount(ctx) - // url, err := SignedURL("bucket", "object", &SignedURLOptions{ - // GoogleAccessID: acc, - // SignBytes: func(b []byte) ([]byte, error) { - // _, signedBytes, err := appengine.SignBytes(ctx, b) - // return signedBytes, err - // }, - // // etc. - // }) + // ctx := appengine.NewContext(request) + // acc, _ := appengine.ServiceAccount(ctx) + // &PostPolicyV4Options{ + // GoogleAccessID: acc, + // SignRawBytes: func(b []byte) ([]byte, error) { + // _, signedBytes, err := appengine.SignBytes(ctx, b) + // return signedBytes, err + // }, + // // etc. + // }) // - // Exactly one of PrivateKey or SignBytes must be non-nil. - SignBytes func(hashBytes []byte) (signature []byte, err error) + // SignRawBytes is equivalent to the SignBytes field on SignedURLOptions; + // that is, you may use the same signing function for the two. + // + // Exactly one of PrivateKey or SignRawBytes must be non-nil. + SignRawBytes func(bytes []byte) (signature []byte, err error) // Expires is the expiration time on the signed URL. // It must be a time in the future. @@ -96,6 +112,23 @@ type PostPolicyV4Options struct { // a 4XX status code, back with the message describing the problem. // Optional. Conditions []PostPolicyV4Condition + + shouldHashSignBytes bool +} + +func (opts *PostPolicyV4Options) clone() *PostPolicyV4Options { + return &PostPolicyV4Options{ + GoogleAccessID: opts.GoogleAccessID, + PrivateKey: opts.PrivateKey, + SignBytes: opts.SignBytes, + SignRawBytes: opts.SignRawBytes, + Expires: opts.Expires, + Style: opts.Style, + Insecure: opts.Insecure, + Fields: opts.Fields, + Conditions: opts.Conditions, + shouldHashSignBytes: opts.shouldHashSignBytes, + } } // PolicyV4Fields describes the attributes for a PostPolicyV4 request. @@ -220,20 +253,22 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options var signingFn func(hashedBytes []byte) ([]byte, error) switch { - case opts.SignBytes != nil: + case opts.SignRawBytes != nil: + signingFn = opts.SignRawBytes + case opts.shouldHashSignBytes: signingFn = opts.SignBytes - case len(opts.PrivateKey) != 0: parsedRSAPrivKey, err := parseKey(opts.PrivateKey) if err != nil { return nil, err } - signingFn = func(hashedBytes []byte) ([]byte, error) { - return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, hashedBytes) + signingFn = func(b []byte) ([]byte, error) { + sum := sha256.Sum256(b) + return rsa.SignPKCS1v15(rand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:]) } default: - return nil, errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") + return nil, errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set") } var descFields PolicyV4Fields @@ -307,10 +342,18 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options } b64Policy := base64.StdEncoding.EncodeToString(condsAsJSON) - shaSum := sha256.Sum256([]byte(b64Policy)) - signature, err := signingFn(shaSum[:]) - if err != nil { - return nil, err + var signature []byte + var signErr error + + if opts.shouldHashSignBytes { + // SignBytes expects hashed bytes as input instead of raw bytes, so we hash them + shaSum := sha256.Sum256([]byte(b64Policy)) + signature, signErr = signingFn(shaSum[:]) + } else { + signature, signErr = signingFn([]byte(b64Policy)) + } + if signErr != nil { + return nil, signErr } policyFields["policy"] = b64Policy @@ -348,15 +391,16 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options // validatePostPolicyV4Options checks that: // * GoogleAccessID is set -// * either but not both PrivateKey and SignBytes are set or nil, but not both -// * Expires, the deadline is not in the past +// * either PrivateKey or SignRawBytes/SignBytes is set, but not both +// * the deadline set in Expires is not in the past // * if Style is not set, it'll use PathStyle +// * sets shouldHashSignBytes to true if opts.SignBytes should be used func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error { if opts == nil || opts.GoogleAccessID == "" { return errors.New("storage: missing required GoogleAccessID") } - if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil; privBlank == signBlank { - return errors.New("storage: exactly one of PrivateKey or SignedBytes must be set") + if privBlank, signBlank := len(opts.PrivateKey) == 0, opts.SignBytes == nil && opts.SignRawBytes == nil; privBlank == signBlank { + return errors.New("storage: exactly one of PrivateKey or SignRawBytes must be set") } if opts.Expires.Before(now) { return errors.New("storage: expecting Expires to be in the future") @@ -364,6 +408,9 @@ func validatePostPolicyV4Options(opts *PostPolicyV4Options, now time.Time) error if opts.Style == nil { opts.Style = PathStyle() } + if opts.SignRawBytes == nil && opts.SignBytes != nil { + opts.shouldHashSignBytes = true + } return nil } diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index 4192c02ba9..86153256c3 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -163,7 +163,7 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) } var res *http.Response - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { res, err = o.c.hc.Do(req) if err != nil { return err @@ -210,7 +210,7 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) gen = gen64 } return nil - }) + }, o.retry, true) if err != nil { return nil, err } @@ -483,7 +483,7 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt var msg *storagepb.ReadObjectResponse var err error - err = runWithRetry(cc, func() error { + err = run(cc, func() error { stream, err = o.c.gc.ReadObject(cc, req) if err != nil { return err @@ -492,7 +492,7 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt msg, err = stream.Recv() return err - }) + }, o.retry, true) if err != nil { // Close the stream context we just created to ensure we don't leak // resources. @@ -541,8 +541,8 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt } // Only support checksums when reading an entire object, not a range. - if msg.GetObjectChecksums().Crc32C != nil && offset == 0 && length == 0 { - r.wantCRC = msg.GetObjectChecksums().GetCrc32C() + if checksums := msg.GetObjectChecksums(); checksums != nil && checksums.Crc32C != nil && offset == 0 && length == 0 { + r.wantCRC = checksums.GetCrc32C() r.checkCRC = true } diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 36969c1eb7..07a5ee077c 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -41,6 +41,7 @@ import ( "cloud.google.com/go/internal/trace" "cloud.google.com/go/internal/version" gapic "cloud.google.com/go/storage/internal/apiv2" + "github.com/googleapis/gax-go/v2" "golang.org/x/oauth2/google" "golang.org/x/xerrors" "google.golang.org/api/googleapi" @@ -50,6 +51,7 @@ import ( "google.golang.org/api/transport" htransport "google.golang.org/api/transport/http" storagepb "google.golang.org/genproto/googleapis/storage/v2" + "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/timestamppb" @@ -81,6 +83,12 @@ const ( // ScopeReadWrite grants permissions to manage your // data in Google Cloud Storage. ScopeReadWrite = raw.DevstorageReadWriteScope + + // defaultConnPoolSize is the default number of connections + // to initialize in the GAPIC gRPC connection pool. A larger + // connection pool may be necessary for jobs that require + // high throughput and/or leverage many concurrent streams. + defaultConnPoolSize = 4 ) var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) @@ -102,6 +110,7 @@ type Client struct { readHost string // May be nil. creds *google.Credentials + retry *retryConfig // gc is an optional gRPC-based, GAPIC client. // @@ -203,11 +212,34 @@ func newHybridClient(ctx context.Context, opts *hybridClientOptions) (*Client, e if opts == nil { opts = &hybridClientOptions{} } + opts.GRPCOpts = append(defaultGRPCOptions(), opts.GRPCOpts...) + c, err := NewClient(ctx, opts.HTTPOpts...) if err != nil { return nil, err } + // Set emulator options for gRPC if an emulator was specified. Note that in a + // hybrid client, STORAGE_EMULATOR_HOST will set the host to use for HTTP and + // STORAGE_EMULATOR_HOST_GRPC will set the host to use for gRPC (when using a + // local emulator, HTTP and gRPC must use different ports, so this is + // necessary). + // TODO: when full gRPC client is available, remove STORAGE_EMULATOR_HOST_GRPC + // and use STORAGE_EMULATOR_HOST for both the HTTP and gRPC based clients. + if host := os.Getenv("STORAGE_EMULATOR_HOST_GRPC"); host != "" { + // Strip the scheme from the emulator host. WithEndpoint does not take a + // scheme for gRPC. + if strings.Contains(host, "://") { + host = strings.SplitN(host, "://", 2)[1] + } + + opts.GRPCOpts = append(opts.GRPCOpts, + option.WithEndpoint(host), + option.WithGRPCDialOption(grpc.WithInsecure()), + option.WithoutAuthentication(), + ) + } + g, err := gapic.NewClient(ctx, opts.GRPCOpts...) if err != nil { return nil, err @@ -217,6 +249,14 @@ func newHybridClient(ctx context.Context, opts *hybridClientOptions) (*Client, e return c, nil } +// defaultGRPCOptions returns a set of the default client options +// for gRPC client initialization. +func defaultGRPCOptions() []option.ClientOption { + return []option.ClientOption{ + option.WithGRPCConnectionPool(defaultConnPoolSize), + } +} + // Close closes the Client. // // Close need not be called at program exit. @@ -836,6 +876,7 @@ type ObjectHandle struct { encryptionKey []byte // AES-256 key userProject string // for requester-pays buckets readCompressed bool // Accept-Encoding: gzip + retry *retryConfig } // ACL provides access to the object's access control list. @@ -899,7 +940,7 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error } var obj *raw.Object setClientHeader(call.Header()) - err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) + err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, true) var e *googleapi.Error if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist @@ -1000,7 +1041,11 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( } var obj *raw.Object setClientHeader(call.Header()) - err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) + var isIdempotent bool + if o.conds != nil && o.conds.MetagenerationMatch != 0 { + isIdempotent = true + } + err = run(ctx, func() error { obj, err = call.Do(); return err }, o.retry, isIdempotent) var e *googleapi.Error if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist @@ -1064,7 +1109,13 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { } // Encryption doesn't apply to Delete. setClientHeader(call.Header()) - err := runWithRetry(ctx, func() error { return call.Do() }) + var isIdempotent bool + // Delete is idempotent if GenerationMatch or Generation have been passed in. + // The default generation is negative to get the latest version of the object. + if (o.conds != nil && o.conds.GenerationMatch != 0) || o.gen >= 0 { + isIdempotent = true + } + err := run(ctx, func() error { return call.Do() }, o.retry, isIdempotent) var e *googleapi.Error if ok := xerrors.As(err, &e); ok && e.Code == http.StatusNotFound { return ErrObjectNotExist @@ -1759,6 +1810,169 @@ func setConditionField(call reflect.Value, name string, value interface{}) bool return true } +// Retryer returns an object handle that is configured with custom retry +// behavior as specified by the options that are passed to it. All operations +// on the new handle will use the customized retry configuration. +// These retry options will merge with the bucket's retryer (if set) for the +// returned handle. Options passed into this method will take precedence over +// retry options on the bucket and client. Note that you must explicitly pass in +// each option you want to override. +func (o *ObjectHandle) Retryer(opts ...RetryOption) *ObjectHandle { + o2 := *o + var retry *retryConfig + if o.retry != nil { + // merge the options with the existing retry + retry = o.retry + } else { + retry = &retryConfig{} + } + for _, opt := range opts { + opt.apply(retry) + } + o2.retry = retry + o2.acl.retry = retry + return &o2 +} + +// SetRetry configures the client with custom retry behavior as specified by the +// options that are passed to it. All operations using this client will use the +// customized retry configuration. +// This should be called once before using the client for network operations, as +// there could be indeterminate behaviour with operations in progress. +// Retry options set on a bucket or object handle will take precedence over +// these options. +func (c *Client) SetRetry(opts ...RetryOption) { + var retry *retryConfig + if c.retry != nil { + // merge the options with the existing retry + retry = c.retry + } else { + retry = &retryConfig{} + } + for _, opt := range opts { + opt.apply(retry) + } + c.retry = retry +} + +// RetryOption allows users to configure non-default retry behavior for API +// calls made to GCS. +type RetryOption interface { + apply(config *retryConfig) +} + +// WithBackoff allows configuration of the backoff timing used for retries. +// Available configuration options (Initial, Max and Multiplier) are described +// at https://pkg.go.dev/github.com/googleapis/gax-go/v2#Backoff. If any fields +// are not supplied by the user, gax default values will be used. +func WithBackoff(backoff gax.Backoff) RetryOption { + return &withBackoff{ + backoff: backoff, + } +} + +type withBackoff struct { + backoff gax.Backoff +} + +func (wb *withBackoff) apply(config *retryConfig) { + config.backoff = &wb.backoff +} + +// RetryPolicy describes the available policies for which operations should be +// retried. The default is `RetryIdempotent`. +type RetryPolicy int + +const ( + // RetryIdempotent causes only idempotent operations to be retried when the + // service returns a transient error. Using this policy, fully idempotent + // operations (such as `ObjectHandle.Attrs()`) will always be retried. + // Conditionally idempotent operations (for example `ObjectHandle.Update()`) + // will be retried only if the necessary conditions have been supplied (in + // the case of `ObjectHandle.Update()` this would mean supplying a + // `Conditions.MetagenerationMatch` condition is required). + RetryIdempotent RetryPolicy = iota + + // RetryAlways causes all operations to be retried when the service returns a + // transient error, regardless of idempotency considerations. + RetryAlways + + // RetryNever causes the client to not perform retries on failed operations. + RetryNever +) + +// WithPolicy allows the configuration of which operations should be performed +// with retries for transient errors. +func WithPolicy(policy RetryPolicy) RetryOption { + return &withPolicy{ + policy: policy, + } +} + +type withPolicy struct { + policy RetryPolicy +} + +func (ws *withPolicy) apply(config *retryConfig) { + config.policy = ws.policy +} + +// WithErrorFunc allows users to pass a custom function to the retryer. Errors +// will be retried if and only if `shouldRetry(err)` returns true. +// By default, the following errors are retried (see invoke.go for the default +// shouldRetry function): +// +// - HTTP responses with codes 408, 429, 502, 503, and 504. +// +// - Transient network errors such as connection reset and io.ErrUnexpectedEOF. +// +// - Errors which are considered transient using the Temporary() interface. +// +// - Wrapped versions of these errors. +// +// This option can be used to retry on a different set of errors than the +// default. +func WithErrorFunc(shouldRetry func(err error) bool) RetryOption { + return &withErrorFunc{ + shouldRetry: shouldRetry, + } +} + +type withErrorFunc struct { + shouldRetry func(err error) bool +} + +func (wef *withErrorFunc) apply(config *retryConfig) { + config.shouldRetry = wef.shouldRetry +} + +type retryConfig struct { + backoff *gax.Backoff + policy RetryPolicy + shouldRetry func(err error) bool +} + +func (r *retryConfig) clone() *retryConfig { + if r == nil { + return nil + } + + var bo *gax.Backoff + if r.backoff != nil { + bo = &gax.Backoff{ + Initial: r.backoff.Initial, + Max: r.backoff.Max, + Multiplier: r.backoff.Multiplier, + } + } + + return &retryConfig{ + backoff: bo, + policy: r.policy, + shouldRetry: r.shouldRetry, + } +} + // composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods // that modifyCall searches for by name. type composeSourceObj struct { @@ -1802,10 +2016,10 @@ func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, r := c.raw.Projects.ServiceAccount.Get(projectID) var res *raw.ServiceAccount var err error - err = runWithRetry(ctx, func() error { + err = run(ctx, func() error { res, err = r.Context(ctx).Do() return err - }) + }, c.retry, true) if err != nil { return "", err } diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index ec55e4a3ba..51d8b63779 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -172,6 +172,22 @@ func (w *Writer) open() error { // call to set up the upload as well as calls to upload individual chunks // for a resumable upload (as long as the chunk size is non-zero). Hence // there is no need to add retries here. + + // Retry only when the operation is idempotent or the retry policy is RetryAlways. + isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true) + var useRetry bool + if (w.o.retry == nil || w.o.retry.policy == RetryIdempotent) && isIdempotent { + useRetry = true + } else if w.o.retry != nil && w.o.retry.policy == RetryAlways { + useRetry = true + } + if useRetry { + if w.o.retry != nil { + call.WithRetry(w.o.retry.backoff, w.o.retry.shouldRetry) + } else { + call.WithRetry(nil, nil) + } + } resp, err = call.Do() } if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index e5f01f7913..f422fb6422 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,7 +11,7 @@ cloud.google.com/go/compute/metadata # cloud.google.com/go/iam v0.1.1 ## explicit; go 1.11 cloud.google.com/go/iam -# cloud.google.com/go/storage v1.18.2 +# cloud.google.com/go/storage v1.19.0 ## explicit; go 1.11 cloud.google.com/go/storage cloud.google.com/go/storage/internal/apiv2