From db833d31939b9ef3819f4378bb1215044f4efa79 Mon Sep 17 00:00:00 2001 From: Charles Dixon Date: Thu, 10 Oct 2019 10:19:35 +0100 Subject: [PATCH] GOCBC-534: Add retry API Motivation ---------- The SDK should retry requests under some circumstances. It should also provide the user with a way to customize this behavior. Changes ------- Added the retry API. Added RetryStrategy to all operations. Updated http service based operations to attempt retries based on responses. Updated timeouts across the board to contain more information about retries. Change-Id: I5dabb488abbdc161258fefeaa24391407d34a1ea Reviewed-on: http://review.couchbase.org/116847 Reviewed-by: Brett Lawson Tested-by: Charles Dixon --- analyticsquery_options.go | 5 +- bucket.go | 16 +- bucket_collectionsmgr.go | 171 ++++++++---- bucket_viewindexes.go | 130 ++++++--- bucket_viewquery.go | 53 ++-- cluster.go | 59 ++-- cluster_analyticsindexes.go | 116 +++++--- cluster_analyticsquery.go | 222 +++++++++------ cluster_analyticsquery_test.go | 110 +++++++- cluster_bucketmgr.go | 181 +++++++++--- cluster_query.go | 239 ++++++++-------- cluster_query_test.go | 70 +++-- cluster_queryindexes.go | 65 +++-- cluster_searchindexes.go | 248 ++++++++++++----- cluster_searchquery.go | 229 ++++++++------- cluster_searchquery_test.go | 16 +- cluster_usermgr.go | 261 ++++++++++++++---- collection_binary_crud.go | 27 ++ collection_bulk.go | 45 ++- collection_crud.go | 133 +++++++-- collection_crud_test.go | 104 ++++++- collection_subdoc.go | 20 +- error.go | 67 +++-- go.mod | 2 +- go.sum | 4 +- mock_test.go | 33 ++- queryoptions.go | 3 +- retry.go | 195 +++++++++++++ retry_test.go | 158 +++++++++++ retrybehaviour.go | 62 ----- retrybehaviour_test.go | 53 ---- searchquery_options.go | 3 +- stateblock.go | 6 +- .../beer_sample_analytics_temp_error.json | 18 ++ viewquery_options.go | 2 + 35 files changed, 2222 insertions(+), 904 deletions(-) create mode 100644 retry.go create mode 100644 retry_test.go delete mode 100644 retrybehaviour.go delete mode 100644 retrybehaviour_test.go create mode 100644 testdata/beer_sample_analytics_temp_error.json diff --git a/analyticsquery_options.go b/analyticsquery_options.go index ccc0e5b8..68a05c56 100644 --- a/analyticsquery_options.go +++ b/analyticsquery_options.go @@ -34,7 +34,8 @@ type AnalyticsOptions struct { // JSONSerializer is used to deserialize each row in the result. This should be a JSON deserializer as results are JSON. // NOTE: if not set then query will always default to DefaultJSONSerializer. - Serializer JSONSerializer + Serializer JSONSerializer + RetryStrategy RetryStrategy } func (opts *AnalyticsOptions) toMap(statement string) (map[string]interface{}, error) { @@ -46,7 +47,7 @@ func (opts *AnalyticsOptions) toMap(statement string) (map[string]interface{}, e } if opts.ClientContextID == "" { - execOpts["client_context_id"] = uuid.New() + execOpts["client_context_id"] = uuid.New().String() } else { execOpts["client_context_id"] = opts.ClientContextID } diff --git a/bucket.go b/bucket.go index 1648ccb1..e7ef1d73 100644 --- a/bucket.go +++ b/bucket.go @@ -29,6 +29,8 @@ func newBucket(sb *stateBlock, bucketName string, opts BucketOptions) *Bucket { Transcoder: sb.Transcoder, Serializer: sb.Serializer, + + RetryStrategyWrapper: sb.RetryStrategyWrapper, }, } } @@ -85,9 +87,10 @@ func (b *Bucket) ViewIndexes() (*ViewIndexManager, error) { } return &ViewIndexManager{ - bucketName: b.Name(), - httpClient: provider, - globalTimeout: b.sb.ManagementTimeout, + bucketName: b.Name(), + httpClient: provider, + globalTimeout: b.sb.ManagementTimeout, + defaultRetryStrategy: b.sb.RetryStrategyWrapper, }, nil } @@ -100,8 +103,9 @@ func (b *Bucket) CollectionManager() (*CollectionManager, error) { } return &CollectionManager{ - httpClient: provider, - bucketName: b.Name(), - globalTimeout: b.sb.ManagementTimeout, + httpClient: provider, + bucketName: b.Name(), + globalTimeout: b.sb.ManagementTimeout, + defaultRetryStrategy: b.sb.RetryStrategyWrapper, }, nil } diff --git a/bucket_collectionsmgr.go b/bucket_collectionsmgr.go index 969baba7..04879560 100644 --- a/bucket_collectionsmgr.go +++ b/bucket_collectionsmgr.go @@ -14,9 +14,10 @@ import ( // CollectionManager provides methods for performing collections management. // Volatile: This API is subject to change at any time. type CollectionManager struct { - httpClient httpProvider - bucketName string - globalTimeout time.Duration + httpClient httpProvider + bucketName string + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // CollectionSpec describes the specification of a collection. @@ -33,8 +34,9 @@ type ScopeSpec struct { // CollectionExistsOptions is the set of options available to the CollectionExists operation. type CollectionExistsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // These 3 types are temporary. They are necessary for now as the server beta was released with ns_server returning @@ -76,14 +78,21 @@ func (cm *CollectionManager) CollectionExists(spec CollectionSpec, opts *Collect defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + posts := url.Values{} posts.Add("name", spec.Name) req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), + Method: "GET", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -152,8 +161,9 @@ func (cm *CollectionManager) CollectionExists(spec CollectionSpec, opts *Collect // ScopeExistsOptions is the set of options available to the ScopeExists operation. type ScopeExistsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // ScopeExists verifies whether or not a scope exists on the bucket. @@ -173,11 +183,18 @@ func (cm *CollectionManager) ScopeExists(scopeName string, opts *ScopeExistsOpti defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), + Method: "GET", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -235,8 +252,9 @@ func (cm *CollectionManager) ScopeExists(scopeName string, opts *ScopeExistsOpti // GetScopeOptions is the set of options available to the GetScope operation. type GetScopeOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetScope gets a scope from the bucket. @@ -256,11 +274,18 @@ func (cm *CollectionManager) GetScope(scopeName string, opts *GetScopeOptions) ( defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), + Method: "GET", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -346,8 +371,9 @@ func (cm *CollectionManager) GetScope(scopeName string, opts *GetScopeOptions) ( // GetAllScopesOptions is the set of options available to the GetAllScopes operation. type GetAllScopesOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllScopes gets all scopes from the bucket. @@ -361,11 +387,18 @@ func (cm *CollectionManager) GetAllScopes(opts *GetAllScopesOptions) ([]ScopeSpe defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), + Method: "GET", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -440,8 +473,9 @@ func (cm *CollectionManager) GetAllScopes(opts *GetAllScopesOptions) ([]ScopeSpe // CreateCollectionOptions is the set of options available to the CreateCollection operation. type CreateCollectionOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // CreateCollection creates a new collection on the bucket. @@ -467,16 +501,22 @@ func (cm *CollectionManager) CreateCollection(spec CollectionSpec, opts *CreateC defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + posts := url.Values{} posts.Add("name", spec.Name) req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s", cm.bucketName, spec.ScopeName), - Method: "POST", - Body: []byte(posts.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s", cm.bucketName, spec.ScopeName), + Method: "POST", + Body: []byte(posts.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -510,8 +550,9 @@ func (cm *CollectionManager) CreateCollection(spec CollectionSpec, opts *CreateC // DropCollectionOptions is the set of options available to the DropCollection operation. type DropCollectionOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropCollection removes a collection. @@ -537,11 +578,17 @@ func (cm *CollectionManager) DropCollection(spec CollectionSpec, opts *DropColle defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s/%s", cm.bucketName, spec.ScopeName, spec.Name), - Method: "DELETE", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s/%s", cm.bucketName, spec.ScopeName, spec.Name), + Method: "DELETE", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -575,8 +622,9 @@ func (cm *CollectionManager) DropCollection(spec CollectionSpec, opts *DropColle // CreateScopeOptions is the set of options available to the CreateScope operation. type CreateScopeOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // CreateScope creates a new scope on the bucket. @@ -596,16 +644,22 @@ func (cm *CollectionManager) CreateScope(scopeName string, opts *CreateScopeOpti defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + posts := url.Values{} posts.Add("name", scopeName) req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), - Method: "POST", - Body: []byte(posts.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections", cm.bucketName), + Method: "POST", + Body: []byte(posts.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := cm.httpClient.DoHttpRequest(req) @@ -639,8 +693,9 @@ func (cm *CollectionManager) CreateScope(scopeName string, opts *CreateScopeOpti // DropScopeOptions is the set of options available to the DropScope operation. type DropScopeOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropScope removes a scope. @@ -654,11 +709,17 @@ func (cm *CollectionManager) DropScope(scopeName string, opts *DropScopeOptions) defer cancel() } + retryStrategy := cm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s", cm.bucketName, scopeName), - Method: "DELETE", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/collections/%s", cm.bucketName, scopeName), + Method: "DELETE", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := cm.httpClient.DoHttpRequest(req) diff --git a/bucket_viewindexes.go b/bucket_viewindexes.go index a7ff0772..32921e8b 100644 --- a/bucket_viewindexes.go +++ b/bucket_viewindexes.go @@ -27,9 +27,10 @@ const ( // ViewIndexManager provides methods for performing View management. // Volatile: This API is subject to change at any time. type ViewIndexManager struct { - bucketName string - httpClient httpProvider - globalTimeout time.Duration + bucketName string + httpClient httpProvider + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // View represents a Couchbase view within a design document. @@ -50,8 +51,9 @@ type DesignDocument struct { // GetDesignDocumentOptions is the set of options available to the ViewIndexManager GetDesignDocument operation. type GetDesignDocumentOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } func (vm *ViewIndexManager) ddocName(name string, isProd DesignDocumentNamespace) string { @@ -81,15 +83,30 @@ func (vm *ViewIndexManager) GetDesignDocument(name string, namespace DesignDocum name = vm.ddocName(name, namespace) + retryStrategy := vm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(CapiService), - Path: fmt.Sprintf("/_design/%s", name), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(CapiService), + Path: fmt.Sprintf("/_design/%s", name), + Method: "GET", + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := vm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -123,8 +140,9 @@ func (vm *ViewIndexManager) GetDesignDocument(name string, namespace DesignDocum // GetAllDesignDocumentsOptions is the set of options available to the ViewIndexManager GetAllDesignDocuments operation. type GetAllDesignDocumentsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllDesignDocuments will retrieve all design documents for the given bucket. @@ -138,15 +156,30 @@ func (vm *ViewIndexManager) GetAllDesignDocuments(namespace DesignDocumentNamesp defer cancel() } + retryStrategy := vm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/ddocs", vm.bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/ddocs", vm.bucketName), + Method: "GET", + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := vm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -193,8 +226,9 @@ func (vm *ViewIndexManager) GetAllDesignDocuments(namespace DesignDocumentNamesp // UpsertDesignDocumentOptions is the set of options available to the ViewIndexManager UpsertDesignDocument operation. type UpsertDesignDocumentOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // UpsertDesignDocument will insert a design document to the given bucket, or update @@ -216,16 +250,30 @@ func (vm *ViewIndexManager) UpsertDesignDocument(ddoc DesignDocument, namespace ddoc.Name = vm.ddocName(ddoc.Name, namespace) + retryStrategy := vm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(CapiService), - Path: fmt.Sprintf("/_design/%s", ddoc.Name), - Method: "PUT", - Body: data, - Context: ctx, + Service: gocbcore.ServiceType(CapiService), + Path: fmt.Sprintf("/_design/%s", ddoc.Name), + Method: "PUT", + Body: data, + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := vm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -246,8 +294,9 @@ func (vm *ViewIndexManager) UpsertDesignDocument(ddoc DesignDocument, namespace // DropDesignDocumentOptions is the set of options available to the ViewIndexManager Upsert operation. type DropDesignDocumentOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropDesignDocument will remove a design document from the given bucket. @@ -263,15 +312,29 @@ func (vm *ViewIndexManager) DropDesignDocument(name string, namespace DesignDocu name = vm.ddocName(name, namespace) + retryStrategy := vm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(CapiService), - Path: fmt.Sprintf("/_design/%s", name), - Method: "DELETE", - Context: ctx, + Service: gocbcore.ServiceType(CapiService), + Path: fmt.Sprintf("/_design/%s", name), + Method: "DELETE", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := vm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -296,8 +359,9 @@ func (vm *ViewIndexManager) DropDesignDocument(name string, namespace DesignDocu // PublishDesignDocumentOptions is the set of options available to the ViewIndexManager PublishDesignDocument operation. type PublishDesignDocumentOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // PublishDesignDocument publishes a design document to the given bucket. @@ -312,7 +376,8 @@ func (vm *ViewIndexManager) PublishDesignDocument(name string, opts *PublishDesi } devdoc, err := vm.GetDesignDocument(name, false, &GetDesignDocumentOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { indexErr, ok := err.(viewIndexError) @@ -325,7 +390,8 @@ func (vm *ViewIndexManager) PublishDesignDocument(name string, opts *PublishDesi } err = vm.UpsertDesignDocument(*devdoc, true, &UpsertDesignDocumentOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { return errors.Wrap(err, "failed to create ") diff --git a/bucket_viewquery.go b/bucket_viewquery.go index f351b2a4..c0ccfa93 100644 --- a/bucket_viewquery.go +++ b/bucket_viewquery.go @@ -322,7 +322,12 @@ func (b *Bucket) ViewQuery(designDoc string, viewName string, opts *ViewOptions) opts.Serializer = b.sb.Serializer } - res, err := b.executeViewQuery(ctx, "_view", designDoc, viewName, *urlValues, provider, cancel, opts.Serializer) + wrapper := b.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + wrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + + res, err := b.executeViewQuery(ctx, "_view", designDoc, viewName, *urlValues, provider, cancel, opts.Serializer, wrapper) if err != nil { cancel() return nil, err @@ -332,13 +337,16 @@ func (b *Bucket) ViewQuery(designDoc string, viewName string, opts *ViewOptions) } func (b *Bucket) executeViewQuery(ctx context.Context, viewType, ddoc, viewName string, - options url.Values, provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*ViewResult, error) { + options url.Values, provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer, + wrapper *retryStrategyWrapper) (*ViewResult, error) { reqUri := fmt.Sprintf("/_design/%s/%s/%s?%s", ddoc, viewType, viewName, options.Encode()) req := &gocbcore.HttpRequest{ - Service: gocbcore.CapiService, - Path: reqUri, - Method: "GET", - Context: ctx, + Service: gocbcore.CapiService, + Path: reqUri, + Method: "GET", + Context: ctx, + IsIdempotent: true, + RetryStrategy: wrapper, } resp, err := provider.DoHttpRequest(req) @@ -347,12 +355,15 @@ func (b *Bucket) executeViewQuery(ctx context.Context, viewType, ddoc, viewName return nil, serviceNotAvailableError{message: gocbcore.ErrNoCapiService.Error()} } - // as we're effectively manually timing out the request using cancellation we need - // to check if the original context has timed out as err itself will only show as canceled - if ctx.Err() == context.DeadlineExceeded { - return nil, timeoutError{} + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.Identifier(), + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } } - return nil, errors.Wrap(err, "could not complete query http request") + + return nil, err } queryResults := &ViewResult{ @@ -362,24 +373,32 @@ func (b *Bucket) executeViewQuery(ctx context.Context, viewType, ddoc, viewName if resp.StatusCode == 500 { // We have to handle the views 500 case as a special case because the body can be of form [] or {} defer func() { - err := resp.Body.Close() - if err != nil { - logDebugf("Failed to close socket (%s)", err.Error()) - } }() decoder := json.NewDecoder(resp.Body) t, err := decoder.Token() if err != nil { + err := resp.Body.Close() + if err != nil { + logDebugf("Failed to close socket (%s)", err.Error()) + } return nil, err } delim, ok := t.(json.Delim) if !ok { + err := resp.Body.Close() + if err != nil { + logDebugf("Failed to close socket (%s)", err.Error()) + } return nil, clientError{message: "could not read response body, no data found"} } if delim == '[' { errMsg, err := decoder.Token() if err != nil { + err := resp.Body.Close() + if err != nil { + logDebugf("Failed to close socket (%s)", err.Error()) + } return nil, err } err = viewMultiError{ @@ -416,6 +435,7 @@ func (b *Bucket) executeViewQuery(ctx context.Context, viewType, ddoc, viewName if bodyErr != nil { logDebugf("Failed to close socket (%s)", bodyErr.Error()) } + return nil, err } @@ -430,7 +450,8 @@ func (b *Bucket) executeViewQuery(ctx context.Context, viewType, ddoc, viewName logDebugf("Failed to close response body, %s", bodyErr.Error()) } - // There are no rows and there are errors so fast fail + // No retries for views + err = queryResults.makeError() if err != nil { return nil, err diff --git a/cluster.go b/cluster.go index 63224107..bed33418 100644 --- a/cluster.go +++ b/cluster.go @@ -46,6 +46,7 @@ type ClusterOptions struct { // will default to DefaultJSONSerializer. NOTE: This is entirely independent of Transcoder. Serializer JSONSerializer DisableMutationTokens bool + RetryStrategy RetryStrategy } // ClusterCloseOptions is the set of options available when disconnecting from a Cluster. @@ -120,27 +121,28 @@ func Connect(connStr string, opts ClusterOptions) (*Cluster, error) { if opts.Serializer == nil { opts.Serializer = &DefaultJSONSerializer{} } + if opts.RetryStrategy == nil { + opts.RetryStrategy = NewBestEffortRetryStrategy(nil) + } cluster := &Cluster{ cSpec: connSpec, auth: opts.Authenticator, connections: make(map[string]client), sb: stateBlock{ - ConnectTimeout: connectTimeout, - N1qlRetryBehavior: standardDelayRetryBehavior(10, 2, 500*time.Millisecond, exponentialDelayFunction), - AnalyticsRetryBehavior: standardDelayRetryBehavior(10, 2, 500*time.Millisecond, exponentialDelayFunction), - SearchRetryBehavior: standardDelayRetryBehavior(10, 2, 500*time.Millisecond, exponentialDelayFunction), - QueryTimeout: queryTimeout, - AnalyticsTimeout: analyticsTimeout, - SearchTimeout: searchTimeout, - ViewTimeout: viewTimeout, - KvTimeout: kvTimeout, - DuraTimeout: 40000 * time.Millisecond, - DuraPollTimeout: 100 * time.Millisecond, - Transcoder: opts.Transcoder, - Serializer: opts.Serializer, - UseMutationTokens: !opts.DisableMutationTokens, - ManagementTimeout: managementTimeout, + ConnectTimeout: connectTimeout, + QueryTimeout: queryTimeout, + AnalyticsTimeout: analyticsTimeout, + SearchTimeout: searchTimeout, + ViewTimeout: viewTimeout, + KvTimeout: kvTimeout, + DuraTimeout: 40000 * time.Millisecond, + DuraPollTimeout: 100 * time.Millisecond, + Transcoder: opts.Transcoder, + Serializer: opts.Serializer, + UseMutationTokens: !opts.DisableMutationTokens, + ManagementTimeout: managementTimeout, + RetryStrategyWrapper: newRetryStrategyWrapper(opts.RetryStrategy), }, queryCache: make(map[string]*n1qlCache), @@ -428,8 +430,9 @@ func (c *Cluster) Users() (*UserManager, error) { } return &UserManager{ - httpClient: provider, - globalTimeout: c.sb.ManagementTimeout, + httpClient: provider, + globalTimeout: c.sb.ManagementTimeout, + defaultRetryStrategy: c.sb.RetryStrategyWrapper, }, nil } @@ -442,8 +445,9 @@ func (c *Cluster) Buckets() (*BucketManager, error) { } return &BucketManager{ - httpClient: provider, - globalTimeout: c.sb.ManagementTimeout, + httpClient: provider, + globalTimeout: c.sb.ManagementTimeout, + defaultRetryStrategy: c.sb.RetryStrategyWrapper, }, nil } @@ -455,9 +459,10 @@ func (c *Cluster) AnalyticsIndexes() (*AnalyticsIndexManager, error) { return nil, err } return &AnalyticsIndexManager{ - httpClient: provider, - executeQuery: c.AnalyticsQuery, - globalTimeout: c.sb.ManagementTimeout, + httpClient: provider, + executeQuery: c.AnalyticsQuery, + globalTimeout: c.sb.ManagementTimeout, + defaultRetryStrategy: c.sb.RetryStrategyWrapper, }, nil } @@ -465,8 +470,9 @@ func (c *Cluster) AnalyticsIndexes() (*AnalyticsIndexManager, error) { // Volatile: This API is subject to change at any time. func (c *Cluster) QueryIndexes() (*QueryIndexManager, error) { return &QueryIndexManager{ - executeQuery: c.Query, - globalTimeout: c.sb.ManagementTimeout, + executeQuery: c.Query, + globalTimeout: c.sb.ManagementTimeout, + defaultRetryStrategy: c.sb.RetryStrategyWrapper, }, nil } @@ -477,7 +483,8 @@ func (c *Cluster) SearchIndexes() (*SearchIndexManager, error) { return nil, err } return &SearchIndexManager{ - httpClient: provider, - globalTimeout: c.sb.ManagementTimeout, + httpClient: provider, + globalTimeout: c.sb.ManagementTimeout, + defaultRetryStrategy: c.sb.RetryStrategyWrapper, }, nil } diff --git a/cluster_analyticsindexes.go b/cluster_analyticsindexes.go index 5041c42c..caf44ac2 100644 --- a/cluster_analyticsindexes.go +++ b/cluster_analyticsindexes.go @@ -14,9 +14,10 @@ import ( // AnalyticsIndexManager provides methods for performing Couchbase Analytics index management. // Volatile: This API is subject to change at any time. type AnalyticsIndexManager struct { - httpClient httpProvider - executeQuery func(statement string, opts *AnalyticsOptions) (*AnalyticsResult, error) - globalTimeout time.Duration + httpClient httpProvider + executeQuery func(statement string, opts *AnalyticsOptions) (*AnalyticsResult, error) + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // AnalyticsDataset contains information about an analytics dataset, @@ -37,8 +38,9 @@ type AnalyticsIndex struct { // CreateAnalyticsDataverseOptions is the set of options available to the AnalyticsManager CreateDataverse operation. type CreateAnalyticsDataverseOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool } @@ -67,7 +69,8 @@ func (am *AnalyticsIndexManager) CreateDataverse(dataverseName string, opts *Cre q := fmt.Sprintf("CREATE DATAVERSE `%s` %s", dataverseName, ignoreStr) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -86,8 +89,9 @@ func (am *AnalyticsIndexManager) CreateDataverse(dataverseName string, opts *Cre // DropAnalyticsDataverseOptions is the set of options available to the AnalyticsManager DropDataverse operation. type DropAnalyticsDataverseOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool } @@ -110,7 +114,8 @@ func (am *AnalyticsIndexManager) DropDataverse(dataverseName string, opts *DropA q := fmt.Sprintf("DROP DATAVERSE %s %s", dataverseName, ignoreStr) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -129,8 +134,9 @@ func (am *AnalyticsIndexManager) DropDataverse(dataverseName string, opts *DropA // CreateAnalyticsDatasetOptions is the set of options available to the AnalyticsManager CreateDataset operation. type CreateAnalyticsDatasetOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool // Condition can be used to set the WHERE clause for the dataset creation. @@ -176,7 +182,8 @@ func (am *AnalyticsIndexManager) CreateDataset(datasetName, bucketName string, o q := fmt.Sprintf("CREATE DATASET %s %s ON `%s` %s", ignoreStr, datasetName, bucketName, where) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -195,8 +202,9 @@ func (am *AnalyticsIndexManager) CreateDataset(datasetName, bucketName string, o // DropAnalyticsDatasetOptions is the set of options available to the AnalyticsManager DropDataset operation. type DropAnalyticsDatasetOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool DataverseName string @@ -226,7 +234,8 @@ func (am *AnalyticsIndexManager) DropDataset(datasetName string, opts *DropAnaly q := fmt.Sprintf("DROP DATASET %s %s", datasetName, ignoreStr) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -245,8 +254,9 @@ func (am *AnalyticsIndexManager) DropDataset(datasetName string, opts *DropAnaly // GetAllAnalyticsDatasetsOptions is the set of options available to the AnalyticsManager GetAllDatasets operation. type GetAllAnalyticsDatasetsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllDatasets gets all analytics datasets. @@ -263,7 +273,9 @@ func (am *AnalyticsIndexManager) GetAllDatasets(opts *GetAllAnalyticsDatasetsOpt result, err := am.executeQuery( "SELECT d.* FROM Metadata.`Dataset` d WHERE d.DataverseName <> \"Metadata\"", &AnalyticsOptions{ - Context: ctx, + Context: ctx, + ReadOnly: true, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -293,8 +305,9 @@ func (am *AnalyticsIndexManager) GetAllDatasets(opts *GetAllAnalyticsDatasetsOpt // CreateAnalyticsIndexOptions is the set of options available to the AnalyticsManager CreateIndex operation. type CreateAnalyticsIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool DataverseName string @@ -340,7 +353,8 @@ func (am *AnalyticsIndexManager) CreateIndex(datasetName, indexName string, fiel q := fmt.Sprintf("CREATE INDEX `%s` %s ON %s (%s)", indexName, ignoreStr, datasetName, strings.Join(indexFields, ",")) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -359,8 +373,9 @@ func (am *AnalyticsIndexManager) CreateIndex(datasetName, indexName string, fiel // DropAnalyticsIndexOptions is the set of options available to the AnalyticsManager DropIndex operation. type DropAnalyticsIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool DataverseName string @@ -390,7 +405,8 @@ func (am *AnalyticsIndexManager) DropIndex(datasetName, indexName string, opts * q := fmt.Sprintf("DROP INDEX %s.%s %s", datasetName, indexName, ignoreStr) result, err := am.executeQuery(q, &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -409,8 +425,9 @@ func (am *AnalyticsIndexManager) DropIndex(datasetName, indexName string, opts * // GetAllAnalyticsIndexesOptions is the set of options available to the AnalyticsManager GetAllIndexes operation. type GetAllAnalyticsIndexesOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllIndexes gets all analytics indexes. @@ -427,7 +444,9 @@ func (am *AnalyticsIndexManager) GetAllIndexes(opts *GetAllAnalyticsIndexesOptio result, err := am.executeQuery( "SELECT d.* FROM Metadata.`Index` d WHERE d.DataverseName <> \"Metadata\"", &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, + ReadOnly: true, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -457,8 +476,9 @@ func (am *AnalyticsIndexManager) GetAllIndexes(opts *GetAllAnalyticsIndexesOptio // ConnectAnalyticsLinkOptions is the set of options available to the AnalyticsManager ConnectLink operation. type ConnectAnalyticsLinkOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy // Name of the link, if empty defaults to Local LinkName string } @@ -481,7 +501,8 @@ func (am *AnalyticsIndexManager) ConnectLink(opts *ConnectAnalyticsLinkOptions) result, err := am.executeQuery( fmt.Sprintf("CONNECT LINK %s", opts.LinkName), &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -500,8 +521,9 @@ func (am *AnalyticsIndexManager) ConnectLink(opts *ConnectAnalyticsLinkOptions) // DisconnectAnalyticsLinkOptions is the set of options available to the AnalyticsManager DisconnectLink operation. type DisconnectAnalyticsLinkOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy // Name of the link, if empty defaults to Local LinkName string } @@ -524,7 +546,8 @@ func (am *AnalyticsIndexManager) DisconnectLink(opts *DisconnectAnalyticsLinkOpt result, err := am.executeQuery( fmt.Sprintf("DISCONNECT LINK %s", opts.LinkName), &AnalyticsOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { aErr, ok := err.(AnalyticsQueryError) @@ -543,8 +566,9 @@ func (am *AnalyticsIndexManager) DisconnectLink(opts *DisconnectAnalyticsLinkOpt // GetPendingMutationsAnalyticsOptions is the set of options available to the user manager GetPendingMutations operation. type GetPendingMutationsAnalyticsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetPendingMutations returns the number of pending mutations for all indexes in the form of dataverse.dataset:mutations. @@ -558,15 +582,29 @@ func (am *AnalyticsIndexManager) GetPendingMutations(opts *GetPendingMutationsAn defer cancel() } + retryStrategy := am.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(AnalyticsService), - Method: "GET", - Path: fmt.Sprintf("/analytics/node/agg/stats/remaining"), - Context: ctx, + Service: gocbcore.ServiceType(AnalyticsService), + Method: "GET", + Path: fmt.Sprintf("/analytics/node/agg/stats/remaining"), + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := am.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } diff --git a/cluster_analyticsquery.go b/cluster_analyticsquery.go index 4cbc8a39..45dc4e6b 100644 --- a/cluster_analyticsquery.go +++ b/cluster_analyticsquery.go @@ -7,6 +7,8 @@ import ( "strconv" "time" + "github.com/google/uuid" + gocbcore "github.com/couchbase/gocbcore/v8" "github.com/pkg/errors" ) @@ -124,9 +126,19 @@ func (r *AnalyticsResult) Close() error { r.cancel() } if ctxErr == context.DeadlineExceeded { - return timeoutError{} + return timeoutError{ + operationID: r.metadata.clientContextID, + } } if r.err != nil { + if qErr, ok := r.err.(AnalyticsQueryError); ok { + if qErr.Code() == 21002 { + return timeoutError{ + operationID: r.metadata.clientContextID, + } + } + } + return r.err } return err @@ -292,20 +304,16 @@ func (c *Cluster) AnalyticsQuery(statement string, opts *AnalyticsOptions) (*Ana if opts == nil { opts = &AnalyticsOptions{} } - ctx := opts.Context - if ctx == nil { - ctx = context.Background() - } provider, err := c.getHTTPProvider() if err != nil { return nil, err } - return c.analyticsQuery(ctx, statement, opts, provider) + return c.analyticsQuery(statement, opts, provider) } -func (c *Cluster) analyticsQuery(ctx context.Context, statement string, opts *AnalyticsOptions, +func (c *Cluster) analyticsQuery(statement string, opts *AnalyticsOptions, provider httpProvider) (*AnalyticsResult, error) { queryOpts, err := opts.toMap(statement) @@ -322,15 +330,11 @@ func (c *Cluster) analyticsQuery(ctx context.Context, statement string, opts *An } } - if ctx == nil { - ctx = context.Background() + if opts.Context == nil { + opts.Context = context.Background() } - // We need to try to create the context with timeout + 1 second so that the server closes the connection rather - // than us. This is just a better user experience. - timeoutPlusBuffer := timeout + time.Second - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, timeoutPlusBuffer) + ctx, cancel := context.WithTimeout(opts.Context, timeout) now := time.Now() d, _ := ctx.Deadline() @@ -348,22 +352,12 @@ func (c *Cluster) analyticsQuery(ctx context.Context, statement string, opts *An opts.Serializer = c.sb.Serializer } - var retries uint - var res *AnalyticsResult - for { - retries++ - res, err = c.executeAnalyticsQuery(ctx, queryOpts, provider, cancel, opts.Serializer) - if err == nil { - break - } - - if !IsRetryableError(err) || c.sb.AnalyticsRetryBehavior == nil || !c.sb.AnalyticsRetryBehavior.CanRetry(retries) { - break - } - - time.Sleep(c.sb.AnalyticsRetryBehavior.NextInterval(retries)) + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) } + res, err := c.executeAnalyticsQuery(ctx, queryOpts, provider, cancel, opts.ReadOnly, opts.Serializer, retryWrapper) if err != nil { // only cancel on error, if we cancel when things have gone to plan then we'll prematurely close the stream if cancel != nil { @@ -376,7 +370,8 @@ func (c *Cluster) analyticsQuery(ctx context.Context, statement string, opts *An } func (c *Cluster) executeAnalyticsQuery(ctx context.Context, opts map[string]interface{}, - provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*AnalyticsResult, error) { + provider httpProvider, cancel context.CancelFunc, idempotent bool, serializer JSONSerializer, + retryWrapper *retryStrategyWrapper) (*AnalyticsResult, error) { // priority is sent as a header not in the body priority, priorityCastOK := opts["priority"].(int) if priorityCastOK { @@ -389,11 +384,21 @@ func (c *Cluster) executeAnalyticsQuery(ctx context.Context, opts map[string]int } req := &gocbcore.HttpRequest{ - Service: gocbcore.CbasService, - Path: "/analytics/service", - Method: "POST", - Context: ctx, - Body: reqJSON, + Service: gocbcore.CbasService, + Path: "/analytics/service", + Method: "POST", + Context: ctx, + Body: reqJSON, + IsIdempotent: idempotent, + RetryStrategy: retryWrapper, + } + + contextID, ok := opts["client_context_id"].(string) + if ok { + req.UniqueId = contextID + } else { + req.UniqueId = uuid.New().String() + logWarnf("Failed to assert analytics options client_context_id to string. Replacing with %s", req.UniqueId) } if priorityCastOK { @@ -401,66 +406,119 @@ func (c *Cluster) executeAnalyticsQuery(ctx context.Context, opts map[string]int req.Headers["Analytics-Priority"] = strconv.Itoa(priority) } - resp, err := provider.DoHttpRequest(req) - if err != nil { - if err == gocbcore.ErrNoCbasService { - return nil, serviceNotAvailableError{message: gocbcore.ErrNoCbasService.Error()} + for { + resp, err := provider.DoHttpRequest(req) + if err != nil { + if err == gocbcore.ErrNoCbasService { + return nil, serviceNotAvailableError{message: gocbcore.ErrNoCbasService.Error()} + } + + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.Identifier(), + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + + return nil, err } - // as we're effectively manually timing out the request using cancellation we need - // to check if the original context has timed out as err itself will only show as canceled - if ctx.Err() == context.DeadlineExceeded { - return nil, timeoutError{} + epInfo, err := url.Parse(resp.Endpoint) + if err != nil { + logWarnf("Failed to parse N1QL source address") + epInfo = &url.URL{ + Host: "", + } } - return nil, errors.Wrap(err, "could not complete analytics http request") - } - epInfo, err := url.Parse(resp.Endpoint) - if err != nil { - logWarnf("Failed to parse N1QL source address") - epInfo = &url.URL{ - Host: "", + results := &AnalyticsResult{ + metadata: AnalyticsMetadata{ + sourceAddr: epInfo.Host, + }, + httpStatus: resp.StatusCode, + httpProvider: provider, + serializer: serializer, } - } - queryResults := &AnalyticsResult{ - metadata: AnalyticsMetadata{ - sourceAddr: epInfo.Host, - }, - httpStatus: resp.StatusCode, - httpProvider: provider, - serializer: serializer, - } + streamResult, err := newStreamingResults(resp.Body, results.readAttribute) + if err != nil { + return nil, err + } - streamResult, err := newStreamingResults(resp.Body, queryResults.readAttribute) - if err != nil { - return nil, err - } + err = streamResult.readAttributes() + if err != nil { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close socket (%s)", bodyErr.Error()) + } - err = streamResult.readAttributes() - if err != nil { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close socket (%s)", bodyErr.Error()) + return nil, err } - return nil, err - } - queryResults.streamResult = streamResult - - if streamResult.HasRows() { - queryResults.cancel = cancel - queryResults.ctx = ctx - } else { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close response body, %s", bodyErr.Error()) + results.streamResult = streamResult + + if streamResult.HasRows() { + results.cancel = cancel + results.ctx = ctx + } else { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close response body, %s", bodyErr.Error()) + } + + // If this isn't retryable then return immediately, otherwise attempt a retry. If that fails then return + // immediately. + if IsRetryableError(results.err) { + shouldRetry, retryErr := shouldRetryHTTPRequest(ctx, req, gocbcore.ServiceResponseCodeIndicatedRetryReason, + retryWrapper, provider) + if shouldRetry { + continue + } + + if retryErr != nil { + return nil, retryErr + } + } + + // If there are no rows then must be an error but we'll just make sure that users can't ever + // end up in a state where result and error is nil. + if results.err == nil { + return nil, errors.New("Unknown error") + } + return nil, results.err } - // There are no rows and there are errors so fast fail - if queryResults.err != nil { - return nil, queryResults.err + return results, nil + } +} + +func shouldRetryHTTPRequest(ctx context.Context, req *gocbcore.HttpRequest, reason gocbcore.RetryReason, + retryWrapper *retryStrategyWrapper, provider httpProvider) (bool, error) { + waitCh := make(chan struct{}) + retried := provider.MaybeRetryRequest(req, reason, retryWrapper, func() { + waitCh <- struct{}{} + }) + if retried { + select { + case <-waitCh: + return true, nil + case <-ctx.Done(): + if req.CancelRetry() { + // Read the channel so that we don't leave it hanging + <-waitCh + } + + if ctx.Err() == context.DeadlineExceeded { + return false, timeoutError{ + operationID: req.Identifier(), + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + + return false, ctx.Err() } } - return queryResults, nil + return false, nil } diff --git a/cluster_analyticsquery_test.go b/cluster_analyticsquery_test.go index db556aa8..970d823a 100644 --- a/cluster_analyticsquery_test.go +++ b/cluster_analyticsquery_test.go @@ -324,6 +324,103 @@ func TestBasicAnalyticsQuery(t *testing.T) { testAssertAnalyticsQueryResult(t, &expectedResult, res, true) } +func TestBasicAnalyticsRetries(t *testing.T) { + statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" + timeout := 60 * time.Second + + dataBytes, err := loadRawTestDataset("beer_sample_analytics_temp_error") + if err != nil { + t.Fatalf("Could not read test dataset: %v", err) + } + + successDataBytes, err := loadRawTestDataset("beer_sample_analytics_dataset") + if err != nil { + t.Fatalf("Could not read test dataset: %v", err) + } + + var retries int + doHTTP := func(req *gocbcore.HttpRequest) (*gocbcore.HttpResponse, error) { + retries++ + + if retries == 3 { + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 200, + Body: &testReadCloser{bytes.NewBuffer(successDataBytes), nil}, + }, nil + } + + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 200, + Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, + }, nil + } + + provider := &mockHTTPProvider{ + doFn: doHTTP, + } + + cluster := testGetClusterForHTTP(provider, 0, timeout, 0) + + _, err = cluster.AnalyticsQuery(statement, nil) + if err != nil { + t.Fatalf("Expected query execution to not error %v", err) + } + + if retries != 3 { + t.Fatalf("Expected query to be retried 3 time but ws retried %d times", retries) + } +} + +func TestBasicAnalyticsRetriesTimeout(t *testing.T) { + statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" + timeout := 60 * time.Second + + dataBytes, err := loadRawTestDataset("beer_sample_analytics_temp_error") + if err != nil { + t.Fatalf("Could not read test dataset: %v", err) + } + + var retries int + doHTTP := func(req *gocbcore.HttpRequest) (*gocbcore.HttpResponse, error) { + retries++ + + if retries == 3 { + return nil, context.DeadlineExceeded + } + + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 200, + Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, + }, nil + } + + provider := &mockHTTPProvider{ + doFn: doHTTP, + } + + cluster := testGetClusterForHTTP(provider, 0, timeout, 0) + + _, err = cluster.AnalyticsQuery(statement, &AnalyticsOptions{ + ClientContextID: "contextID", + }) + + if !IsTimeoutError(err) { + t.Fatalf("Expected query execution to timeout error %v", err) + } + + if retries != 3 { + t.Fatalf("Expected query to be retried 3 time but ws retried %d times", retries) + } + + tErr := err.(TimeoutErrorWithDetail) + if tErr.OperationID() != "contextID" { + t.Fatalf("Expected OperationID to be contextID but was %s", tErr.OperationID()) + } +} + func TestBasicAnalyticsQuerySerializer(t *testing.T) { dataBytes, err := loadRawTestDataset("beer_sample_query_dataset") if err != nil { @@ -436,7 +533,7 @@ func TestAnalyticsQueryServiceNotFound(t *testing.T) { statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" timeout := 60 * time.Second - cluster := testGetClusterForHTTP(provider, timeout, 0, 0) + cluster := testGetClusterForHTTP(provider, 0, timeout, 0) res, err := cluster.AnalyticsQuery(statement, nil) if err == nil { @@ -503,6 +600,7 @@ func TestAnalyticsQueryClientSideTimeout(t *testing.T) { } } +// If a server side timeout occurs for some reason then we should retry until the context/timeout value is met. func TestAnalyticsQueryStreamTimeout(t *testing.T) { dataBytes, err := loadRawTestDataset("analytics_timeout") if err != nil { @@ -515,7 +613,10 @@ func TestAnalyticsQueryStreamTimeout(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() + retries := 0 + doHTTP := func(req *gocbcore.HttpRequest) (*gocbcore.HttpResponse, error) { + retries++ testAssertAnalyticsQueryRequest(t, req) var opts map[string]interface{} @@ -543,7 +644,12 @@ func TestAnalyticsQueryStreamTimeout(t *testing.T) { Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, } - return resp, nil + select { + case <-req.Context.Done(): + return nil, req.Context.Err() + default: + return resp, nil + } } provider := &mockHTTPProvider{ diff --git a/cluster_bucketmgr.go b/cluster_bucketmgr.go index 2740b007..3284632a 100644 --- a/cluster_bucketmgr.go +++ b/cluster_bucketmgr.go @@ -15,8 +15,9 @@ import ( // See BucketManager for methods that allow creating and removing buckets themselves. // Volatile: This API is subject to change at any time. type BucketManager struct { - httpClient httpProvider - globalTimeout time.Duration + httpClient httpProvider + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // BucketType specifies the kind of bucket. @@ -161,8 +162,9 @@ func contextFromMaybeTimeout(ctx context.Context, timeout time.Duration, globalT // GetBucketOptions is the set of options available to the bucket manager GetBucket operation. type GetBucketOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetBucket returns settings for a bucket on the cluster. @@ -176,19 +178,34 @@ func (bm *BucketManager) GetBucket(bucketName string, opts *GetBucketOptions) (* defer cancel() } - return bm.get(ctx, bucketName) + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + + return bm.get(ctx, bucketName, retryStrategy) } -func (bm *BucketManager) get(ctx context.Context, bucketName string) (*BucketSettings, error) { +func (bm *BucketManager) get(ctx context.Context, bucketName string, strategy *retryStrategyWrapper) (*BucketSettings, error) { req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s", bucketName), - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s", bucketName), + Method: "GET", + Context: ctx, + IsIdempotent: true, + RetryStrategy: strategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -223,8 +240,9 @@ func (bm *BucketManager) get(ctx context.Context, bucketName string) (*BucketSet // GetAllBucketsOptions is the set of options available to the bucket manager GetAll operation. type GetAllBucketsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllBuckets returns a list of all active buckets on the cluster. @@ -238,15 +256,30 @@ func (bm *BucketManager) GetAllBuckets(opts *GetAllBucketsOptions) (map[string]B defer cancel() } + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: "/pools/default/buckets", - Method: "GET", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: "/pools/default/buckets", + Method: "GET", + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -285,8 +318,9 @@ func (bm *BucketManager) GetAllBuckets(opts *GetAllBucketsOptions) (map[string]B // CreateBucketOptions is the set of options available to the bucket manager CreateBucket operation. type CreateBucketOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // CreateBucket creates a bucket on the cluster. @@ -300,6 +334,11 @@ func (bm *BucketManager) CreateBucket(settings CreateBucketSettings, opts *Creat defer cancel() } + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + posts, err := bm.settingsToPostData(&settings.BucketSettings) if err != nil { return err @@ -310,16 +349,25 @@ func (bm *BucketManager) CreateBucket(settings CreateBucketSettings, opts *Creat } req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: "/pools/default/buckets", - Method: "POST", - Body: []byte(posts.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: "/pools/default/buckets", + Method: "POST", + Body: []byte(posts.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -349,8 +397,9 @@ func (bm *BucketManager) CreateBucket(settings CreateBucketSettings, opts *Creat // UpdateBucketOptions is the set of options available to the bucket manager UpdateBucket operation. type UpdateBucketOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // UpdateBucket updates a bucket on the cluster. @@ -364,22 +413,36 @@ func (bm *BucketManager) UpdateBucket(settings BucketSettings, opts *UpdateBucke defer cancel() } + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + posts, err := bm.settingsToPostData(&settings) if err != nil { return err } req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s", settings.Name), - Method: "POST", - Body: []byte(posts.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s", settings.Name), + Method: "POST", + Body: []byte(posts.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -405,8 +468,9 @@ func (bm *BucketManager) UpdateBucket(settings BucketSettings, opts *UpdateBucke // DropBucketOptions is the set of options available to the bucket manager DropBucket operation. type DropBucketOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropBucket will delete a bucket from the cluster by name. @@ -420,15 +484,29 @@ func (bm *BucketManager) DropBucket(name string, opts *DropBucketOptions) error defer cancel() } + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s", name), - Method: "DELETE", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s", name), + Method: "DELETE", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -454,8 +532,9 @@ func (bm *BucketManager) DropBucket(name string, opts *DropBucketOptions) error // FlushBucketOptions is the set of options available to the bucket manager FlushBucket operation. type FlushBucketOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // FlushBucket will delete all the of the data from a bucket. @@ -470,15 +549,29 @@ func (bm *BucketManager) FlushBucket(name string, opts *FlushBucketOptions) erro defer cancel() } + retryStrategy := bm.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Path: fmt.Sprintf("/pools/default/buckets/%s/controller/doFlush", name), - Method: "POST", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Path: fmt.Sprintf("/pools/default/buckets/%s/controller/doFlush", name), + Method: "POST", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := bm.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } diff --git a/cluster_query.go b/cluster_query.go index 16a05021..4bc679a8 100644 --- a/cluster_query.go +++ b/cluster_query.go @@ -127,9 +127,19 @@ func (r *QueryResult) Close() error { r.cancel() } if ctxErr == context.DeadlineExceeded { - return timeoutError{} + return timeoutError{ + operationID: r.metadata.clientContextID, + } } if r.err != nil { + if qErr, ok := r.err.(QueryError); ok { + if qErr.Code() == 1080 { + return timeoutError{ + operationID: r.metadata.clientContextID, + } + } + } + return r.err } return err @@ -297,6 +307,7 @@ func (r *QueryResult) readAttribute(decoder *json.Decoder, t json.Token) (bool, type httpProvider interface { DoHttpRequest(req *gocbcore.HttpRequest) (*gocbcore.HttpResponse, error) + MaybeRetryRequest(req gocbcore.RetryRequest, reason gocbcore.RetryReason, retryStrategy gocbcore.RetryStrategy, retryFunc func()) bool } type clusterCapabilityProvider interface { @@ -314,21 +325,16 @@ func (c *Cluster) Query(statement string, opts *QueryOptions) (*QueryResult, err if opts == nil { opts = &QueryOptions{} } - ctx := opts.Context - if ctx == nil { - ctx = context.Background() - } provider, err := c.getHTTPProvider() if err != nil { return nil, err } - return c.query(ctx, statement, opts, provider) + return c.query(statement, opts, provider) } -func (c *Cluster) query(ctx context.Context, statement string, opts *QueryOptions, - provider httpProvider) (*QueryResult, error) { +func (c *Cluster) query(statement string, opts *QueryOptions, provider httpProvider) (*QueryResult, error) { queryOpts, err := opts.toMap(statement) if err != nil { @@ -345,15 +351,11 @@ func (c *Cluster) query(ctx context.Context, statement string, opts *QueryOption } } - if ctx == nil { - ctx = context.Background() + if opts.Context == nil { + opts.Context = context.Background() } - // We need to try to create the context with timeout + 1 second so that the server closes the connection rather - // than us. This is just a better user experience. - timeoutPlusBuffer := timeout + time.Second - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, timeoutPlusBuffer) + ctx, cancel := context.WithTimeout(opts.Context, timeout) now := time.Now() d, _ := ctx.Deadline() @@ -371,11 +373,16 @@ func (c *Cluster) query(ctx context.Context, statement string, opts *QueryOption opts.Serializer = c.sb.Serializer } + wrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + wrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + var res *QueryResult if opts.AdHoc { - res, err = c.doPreparedN1qlQuery(ctx, queryOpts, provider, cancel, opts.Serializer) + res, err = c.doPreparedN1qlQuery(ctx, queryOpts, provider, cancel, opts.Serializer, wrapper) } else { - res, err = c.doRetryableQuery(ctx, queryOpts, provider, cancel, opts.Serializer) + res, err = c.executeN1qlQuery(ctx, queryOpts, provider, cancel, opts.Serializer, wrapper) } if err != nil { @@ -390,7 +397,7 @@ func (c *Cluster) query(ctx context.Context, statement string, opts *QueryOption } func (c *Cluster) doPreparedN1qlQuery(ctx context.Context, queryOpts map[string]interface{}, - provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*QueryResult, error) { + provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer, wrapper *retryStrategyWrapper) (*QueryResult, error) { if capabilitySupporter, ok := provider.(clusterCapabilityProvider); ok { if !c.supportsEnhancedPreparedStatements() && capabilitySupporter.SupportsClusterCapability(gocbcore.ClusterCapabilityEnhancedPreparedStatements) { @@ -418,21 +425,15 @@ func (c *Cluster) doPreparedN1qlQuery(ctx context.Context, queryOpts map[string] queryOpts["encoded_plan"] = cachedStmt.encodedPlan } - results, err := c.doRetryableQuery(ctx, queryOpts, provider, cancel, serializer) + results, err := c.executeN1qlQuery(ctx, queryOpts, provider, cancel, serializer, wrapper) if err == nil { return results, nil } - - // If we get error 4050, 4070 or 5000, we should attempt - // to re-prepare the statement immediately before failing. - if !IsRetryableError(err) { - return nil, err - } } // Prepare the query if c.supportsEnhancedPreparedStatements() { - results, err := c.prepareEnhancedN1qlQuery(ctx, queryOpts, provider, cancel, serializer) + results, err := c.prepareEnhancedN1qlQuery(ctx, queryOpts, provider, cancel, serializer, wrapper) if err != nil { return nil, err } @@ -445,7 +446,7 @@ func (c *Cluster) doPreparedN1qlQuery(ctx context.Context, queryOpts map[string] } var err error - cachedStmt, err = c.prepareN1qlQuery(ctx, queryOpts, provider) + cachedStmt, err = c.prepareN1qlQuery(ctx, queryOpts, cancel, provider, wrapper) if err != nil { return nil, err } @@ -460,11 +461,11 @@ func (c *Cluster) doPreparedN1qlQuery(ctx context.Context, queryOpts map[string] queryOpts["prepared"] = cachedStmt.name queryOpts["encoded_plan"] = cachedStmt.encodedPlan - return c.doRetryableQuery(ctx, queryOpts, provider, cancel, serializer) + return c.executeN1qlQuery(ctx, queryOpts, provider, cancel, serializer, wrapper) } func (c *Cluster) prepareEnhancedN1qlQuery(ctx context.Context, opts map[string]interface{}, - provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*QueryResult, error) { + provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer, wrapper *retryStrategyWrapper) (*QueryResult, error) { prepOpts := make(map[string]interface{}) for k, v := range opts { @@ -473,11 +474,11 @@ func (c *Cluster) prepareEnhancedN1qlQuery(ctx context.Context, opts map[string] prepOpts["statement"] = "PREPARE " + opts["statement"].(string) prepOpts["auto_execute"] = true - return c.doRetryableQuery(ctx, prepOpts, provider, cancel, serializer) + return c.executeN1qlQuery(ctx, prepOpts, provider, cancel, serializer, wrapper) } func (c *Cluster) prepareN1qlQuery(ctx context.Context, opts map[string]interface{}, - provider httpProvider) (*n1qlCache, error) { + cancel context.CancelFunc, provider httpProvider, wrapper *retryStrategyWrapper) (*n1qlCache, error) { prepOpts := make(map[string]interface{}) for k, v := range opts { @@ -485,9 +486,7 @@ func (c *Cluster) prepareN1qlQuery(ctx context.Context, opts map[string]interfac } prepOpts["statement"] = "PREPARE " + opts["statement"].(string) - // There's no need to pass cancel here, if there's an error then we'll cancel further up the stack - // and if there isn't then we run another query later where we will cancel - prepRes, err := c.doRetryableQuery(ctx, prepOpts, provider, nil, &DefaultJSONSerializer{}) + prepRes, err := c.executeN1qlQuery(ctx, prepOpts, provider, cancel, &DefaultJSONSerializer{}, wrapper) if err != nil { return nil, err } @@ -504,37 +503,6 @@ func (c *Cluster) prepareN1qlQuery(ctx context.Context, opts map[string]interfac }, nil } -func (c *Cluster) doRetryableQuery(ctx context.Context, queryOpts map[string]interface{}, - provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*QueryResult, error) { - var res *QueryResult - var err error - var retries uint - var endpoint string - enhancedStatements := c.supportsEnhancedPreparedStatements() - for { - retries++ - res, err = c.executeN1qlQuery(ctx, queryOpts, provider, cancel, endpoint, serializer) - if err == nil { - break - } - - if !IsRetryableError(err) || c.sb.N1qlRetryBehavior == nil || !c.sb.N1qlRetryBehavior.CanRetry(retries) { - break - } - - if enhancedStatements { - qErr, ok := err.(QueryError) - if ok { - endpoint = qErr.Endpoint() - } - } - - time.Sleep(c.sb.N1qlRetryBehavior.NextInterval(retries)) - } - - return res, err -} - type n1qlPrepData struct { EncodedPlan string `json:"encoded_plan"` Name string `json:"name"` @@ -545,81 +513,112 @@ type n1qlPrepData struct { // settings. This function will inject any additional connection or request-level // settings into the `opts` map. func (c *Cluster) executeN1qlQuery(ctx context.Context, opts map[string]interface{}, - provider httpProvider, cancel context.CancelFunc, endpoint string, serializer JSONSerializer) (*QueryResult, error) { + provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer, wrapper *retryStrategyWrapper) (*QueryResult, error) { reqJSON, err := json.Marshal(opts) if err != nil { return nil, errors.Wrap(err, "failed to marshal query request body") } + readonly, ok := opts["readonly"].(bool) + if !ok { + readonly = false + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.N1qlService, - Path: "/query/service", - Method: "POST", - Context: ctx, - Body: reqJSON, - Endpoint: endpoint, + Service: gocbcore.N1qlService, + Path: "/query/service", + Method: "POST", + Context: ctx, + Body: reqJSON, + IsIdempotent: readonly, + RetryStrategy: wrapper, } - resp, err := provider.DoHttpRequest(req) - if err != nil { - if err == gocbcore.ErrNoN1qlService { - return nil, serviceNotAvailableError{message: gocbcore.ErrNoN1qlService.Error()} + enhancedStatements := c.supportsEnhancedPreparedStatements() + + for { + resp, err := provider.DoHttpRequest(req) + if err != nil { + if err == gocbcore.ErrNoN1qlService { + return nil, serviceNotAvailableError{message: gocbcore.ErrNoN1qlService.Error()} + } + + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.Identifier(), + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + + return nil, errors.Wrap(err, "could not complete query http request") } - // as we're effectively manually timing out the request using cancellation we need - // to check if the original context has timed out as err itself will only show as canceled - if ctx.Err() == context.DeadlineExceeded { - return nil, timeoutError{} + epInfo, err := url.Parse(resp.Endpoint) + if err != nil { + logWarnf("Failed to parse N1QL source address") + epInfo = &url.URL{ + Host: "", + } } - return nil, errors.Wrap(err, "could not complete query http request") - } - epInfo, err := url.Parse(resp.Endpoint) - if err != nil { - logWarnf("Failed to parse N1QL source address") - epInfo = &url.URL{ - Host: "", + results := &QueryResult{ + metadata: QueryMetadata{ + sourceAddr: epInfo.Host, + }, + httpStatus: resp.StatusCode, + serializer: serializer, + enhancedStatements: c.supportsEnhancedPreparedStatements(), } - } - queryResults := &QueryResult{ - metadata: QueryMetadata{ - sourceAddr: epInfo.Host, - }, - httpStatus: resp.StatusCode, - serializer: serializer, - enhancedStatements: c.supportsEnhancedPreparedStatements(), - } + streamResult, err := newStreamingResults(resp.Body, results.readAttribute) + if err != nil { + return nil, err + } - streamResult, err := newStreamingResults(resp.Body, queryResults.readAttribute) - if err != nil { - return nil, err - } + err = streamResult.readAttributes() + if err != nil { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close socket (%s)", bodyErr.Error()) + } - err = streamResult.readAttributes() - if err != nil { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close socket (%s)", bodyErr.Error()) + return nil, err } - return nil, err - } - queryResults.streamResult = streamResult + results.streamResult = streamResult - if streamResult.HasRows() { - queryResults.cancel = cancel - queryResults.ctx = ctx - } else { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close response body, %s", bodyErr.Error()) - } + if streamResult.HasRows() { + results.cancel = cancel + results.ctx = ctx + } else { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close response body, %s", bodyErr.Error()) + } + + if enhancedStatements { + qErr, ok := results.err.(QueryError) + if ok { + req.Endpoint = qErr.Endpoint() + } + } + + if IsRetryableError(results.err) { + shouldRetry, retryErr := shouldRetryHTTPRequest(ctx, req, gocbcore.ServiceResponseCodeIndicatedRetryReason, + wrapper, provider) + if shouldRetry { + continue + } - // There are no rows and there are errors so fast fail - if queryResults.err != nil { - return nil, queryResults.err + if retryErr != nil { + return nil, retryErr + } + } + + return nil, results.err } + + return results, nil } - return queryResults, nil } diff --git a/cluster_query_test.go b/cluster_query_test.go index 4863dada..c2fed14e 100644 --- a/cluster_query_test.go +++ b/cluster_query_test.go @@ -715,7 +715,7 @@ func TestQueryConnectTimeout(t *testing.T) { Timeout: timeout, Context: ctx, }) - if err == nil || !IsTimeoutError(err) { + if !IsTimeoutError(err) { t.Fatal(err) } } @@ -782,7 +782,7 @@ func TestQueryStreamTimeout(t *testing.T) { } err = results.Close() - if err == nil || !IsTimeoutError(err) { + if !IsTimeoutError(err) { t.Fatalf("Error should have been timeout but was %v", err) } } @@ -991,7 +991,7 @@ func testAssertQueryResult(t *testing.T, expectedResult *n1qlResponse, actualRes func TestBasicRetries(t *testing.T) { statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" - timeout := 60 * time.Second + timeout := 20 * time.Millisecond dataBytes, err := loadRawTestDataset("beer_sample_query_temp_error") if err != nil { @@ -1010,11 +1010,16 @@ func TestBasicRetries(t *testing.T) { testAssertQueryRequest(t, req) retries++ - return &gocbcore.HttpResponse{ - Endpoint: "http://localhost:8093", - StatusCode: 503, // this is a guess - Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, - }, nil + select { + case <-req.Context.Done(): + return nil, req.Context.Err() + default: + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 503, + Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, + }, nil + } } provider := &mockHTTPProvider{ @@ -1022,15 +1027,14 @@ func TestBasicRetries(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) _, err = cluster.Query(statement, nil) if err == nil { t.Fatal("Expected query execution to error") } - if retries != 3 { - t.Fatalf("Expected query to be retried 3 time but ws retried %d times", retries) + if retries <= 1 { + t.Fatalf("Expected query to be retried more than once but was retried %d times", retries) } } @@ -1065,7 +1069,6 @@ func TestBasicEnhancedPreparedQuery(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) cluster.queryCache = map[string]*n1qlCache{ "fake": { @@ -1135,7 +1138,6 @@ func TestBasicEnhancedPreparedQueryAlreadySupported(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) cluster.supportsEnhancedStatements = 1 cluster.queryCache = map[string]*n1qlCache{ @@ -1193,7 +1195,6 @@ func TestBasicEnhancedPreparedQueryAlreadyCached(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) cluster.supportsEnhancedStatements = 1 cluster.queryCache = map[string]*n1qlCache{ @@ -1226,7 +1227,7 @@ func TestBasicEnhancedPreparedQueryAlreadyCached(t *testing.T) { func TestBasicRetriesEnhancedPreparedNoRetry(t *testing.T) { statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" - timeout := 60 * time.Second + timeout := 60 * time.Millisecond dataBytes, err := loadRawTestDataset("beer_sample_query_temp_error") if err != nil { @@ -1245,11 +1246,16 @@ func TestBasicRetriesEnhancedPreparedNoRetry(t *testing.T) { testAssertQueryRequest(t, req) retries++ - return &gocbcore.HttpResponse{ - Endpoint: "http://localhost:8093", - StatusCode: 404, - Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, - }, nil + select { + case <-req.Context.Done(): + return nil, req.Context.Err() + default: + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 404, + Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, + }, nil + } } provider := &mockHTTPProvider{ @@ -1260,7 +1266,6 @@ func TestBasicRetriesEnhancedPreparedNoRetry(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) cluster.queryCache = map[string]*n1qlCache{ "fake": { @@ -1289,7 +1294,7 @@ func TestBasicRetriesEnhancedPreparedNoRetry(t *testing.T) { func TestBasicRetriesEnhancedPreparedRetry(t *testing.T) { statement := "select `beer-sample`.* from `beer-sample` WHERE `type` = ? ORDER BY brewery_id, name" - timeout := 60 * time.Second + timeout := 60 * time.Millisecond dataBytes, err := loadRawTestDataset("query_enhanced_statement_temp_error") if err != nil { @@ -1308,11 +1313,16 @@ func TestBasicRetriesEnhancedPreparedRetry(t *testing.T) { testAssertQueryRequest(t, req) retries++ - return &gocbcore.HttpResponse{ - Endpoint: "http://localhost:8093", - StatusCode: 404, - Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, - }, nil + select { + case <-req.Context.Done(): + return nil, req.Context.Err() + default: + return &gocbcore.HttpResponse{ + Endpoint: "http://localhost:8093", + StatusCode: 404, + Body: &testReadCloser{bytes.NewBuffer(dataBytes), nil}, + }, nil + } } provider := &mockHTTPProvider{ @@ -1323,7 +1333,6 @@ func TestBasicRetriesEnhancedPreparedRetry(t *testing.T) { } cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.N1qlRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) cluster.queryCache = map[string]*n1qlCache{ "fake": { @@ -1341,8 +1350,8 @@ func TestBasicRetriesEnhancedPreparedRetry(t *testing.T) { t.Fatal("Expected query execution to error") } - if retries != 3 { - t.Fatalf("Expected query to be retried 3 time but ws retried %d times", retries) + if retries <= 1 { + t.Fatalf("Expected query to be retried more than once but was retried %d times", retries) } if len(cluster.queryCache) != 0 { @@ -1365,6 +1374,7 @@ func testGetClusterForHTTP(provider *mockHTTPProvider, n1qlTimeout, analyticsTim c.sb.QueryTimeout = n1qlTimeout c.sb.AnalyticsTimeout = analyticsTimeout c.sb.SearchTimeout = searchTimeout + c.sb.RetryStrategyWrapper = newRetryStrategyWrapper(NewBestEffortRetryStrategy(nil)) c.sb.Transcoder = NewJSONTranscoder(&DefaultJSONSerializer{}) c.sb.Serializer = &DefaultJSONSerializer{} diff --git a/cluster_queryindexes.go b/cluster_queryindexes.go index 77b79a6b..3714ca3b 100644 --- a/cluster_queryindexes.go +++ b/cluster_queryindexes.go @@ -9,8 +9,9 @@ import ( // QueryIndexManager provides methods for performing Couchbase N1ql index management. // Volatile: This API is subject to change at any time. type QueryIndexManager struct { - executeQuery func(statement string, opts *QueryOptions) (*QueryResult, error) - globalTimeout time.Duration + executeQuery func(statement string, opts *QueryOptions) (*QueryResult, error) + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // QueryIndex represents a Couchbase GSI index. @@ -25,7 +26,8 @@ type QueryIndex struct { } type createQueryIndexOptions struct { - Context context.Context + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool Deferred bool @@ -58,7 +60,8 @@ func (qm *QueryIndexManager) createIndex(bucketName, indexName string, fields [] } rows, err := qm.executeQuery(qs, &QueryOptions{ - Context: opts.Context, + Context: opts.Context, + RetryStrategy: opts.RetryStrategy, }) if err != nil { if strings.Contains(err.Error(), "already exist") { @@ -78,8 +81,9 @@ func (qm *QueryIndexManager) createIndex(bucketName, indexName string, fields [] // CreateQueryIndexOptions is the set of options available to the query indexes CreateIndex operation. type CreateQueryIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool Deferred bool @@ -111,13 +115,15 @@ func (qm *QueryIndexManager) CreateIndex(bucketName, indexName string, fields [] IgnoreIfExists: opts.IgnoreIfExists, Deferred: opts.Deferred, Context: ctx, + RetryStrategy: opts.RetryStrategy, }) } // CreatePrimaryQueryIndexOptions is the set of options available to the query indexes CreatePrimaryIndex operation. type CreatePrimaryQueryIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfExists bool Deferred bool @@ -139,11 +145,13 @@ func (qm *QueryIndexManager) CreatePrimaryIndex(bucketName string, opts *CreateP IgnoreIfExists: opts.IgnoreIfExists, Deferred: opts.Deferred, Context: ctx, + RetryStrategy: opts.RetryStrategy, }) } type dropQueryIndexOptions struct { - Context context.Context + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool } @@ -158,7 +166,8 @@ func (qm *QueryIndexManager) dropIndex(bucketName, indexName string, opts dropQu } rows, err := qm.executeQuery(qs, &QueryOptions{ - Context: opts.Context, + Context: opts.Context, + RetryStrategy: opts.RetryStrategy, }) if err != nil { if strings.Contains(err.Error(), "not found") { @@ -178,8 +187,9 @@ func (qm *QueryIndexManager) dropIndex(bucketName, indexName string, opts dropQu // DropQueryIndexOptions is the set of options available to the query indexes DropIndex operation. type DropQueryIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool } @@ -204,13 +214,15 @@ func (qm *QueryIndexManager) DropIndex(bucketName, indexName string, opts *DropQ return qm.dropIndex(bucketName, indexName, dropQueryIndexOptions{ Context: ctx, IgnoreIfNotExists: opts.IgnoreIfNotExists, + RetryStrategy: opts.RetryStrategy, }) } // DropPrimaryQueryIndexOptions is the set of options available to the query indexes DropPrimaryIndex operation. type DropPrimaryQueryIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy IgnoreIfNotExists bool CustomName string @@ -230,13 +242,15 @@ func (qm *QueryIndexManager) DropPrimaryIndex(bucketName string, opts *DropPrima return qm.dropIndex(bucketName, opts.CustomName, dropQueryIndexOptions{ IgnoreIfNotExists: opts.IgnoreIfNotExists, Context: ctx, + RetryStrategy: opts.RetryStrategy, }) } // GetAllQueryIndexesOptions is the set of options available to the query indexes GetAllIndexes operation. type GetAllQueryIndexesOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllIndexes returns a list of all currently registered indexes. @@ -254,6 +268,8 @@ func (qm *QueryIndexManager) GetAllIndexes(bucketName string, opts *GetAllQueryI queryOpts := &QueryOptions{ Context: ctx, PositionalParameters: []interface{}{bucketName}, + RetryStrategy: opts.RetryStrategy, + ReadOnly: true, } rows, err := qm.executeQuery(q, queryOpts) @@ -276,8 +292,9 @@ func (qm *QueryIndexManager) GetAllIndexes(bucketName string, opts *GetAllQueryI // BuildDeferredQueryIndexOptions is the set of options available to the query indexes BuildDeferredIndexes operation. type BuildDeferredQueryIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // BuildDeferredIndexes builds all indexes which are currently in deferred state. @@ -292,7 +309,8 @@ func (qm *QueryIndexManager) BuildDeferredIndexes(bucketName string, opts *Build } indexList, err := qm.GetAllIndexes(bucketName, &GetAllQueryIndexesOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { return nil, err @@ -322,7 +340,8 @@ func (qm *QueryIndexManager) BuildDeferredIndexes(bucketName string, opts *Build qs += ")" rows, err := qm.executeQuery(qs, &QueryOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { return nil, err @@ -365,7 +384,8 @@ func checkIndexesActive(indexes []QueryIndex, checkList []string) (bool, error) // WatchQueryIndexOptions is the set of options available to the query indexes Watch operation. type WatchQueryIndexOptions struct { - WatchPrimary bool + WatchPrimary bool + RetryStrategy RetryStrategy } // WatchQueryIndexTimeout is used for setting a timeout value for the query indexes WatchIndexes operation. @@ -398,7 +418,8 @@ func (qm *QueryIndexManager) WatchIndexes(bucketName string, watchList []string, curInterval := 50 * time.Millisecond for { indexes, err := qm.GetAllIndexes(bucketName, &GetAllQueryIndexesOptions{ - Context: ctx, + Context: ctx, + RetryStrategy: opts.RetryStrategy, }) if err != nil { return err diff --git a/cluster_searchindexes.go b/cluster_searchindexes.go index 2168f69c..a2a0e0a2 100644 --- a/cluster_searchindexes.go +++ b/cluster_searchindexes.go @@ -13,8 +13,9 @@ import ( // SearchIndexManager provides methods for performing Couchbase FTS index management. // Experimental: This API is subject to change at any time. type SearchIndexManager struct { - httpClient httpProvider - globalTimeout time.Duration + httpClient httpProvider + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } type searchIndexDefs struct { @@ -34,8 +35,9 @@ type searchIndexesResp struct { // GetAllSearchIndexOptions is the set of options available to the search indexes GetAllIndexes operation. type GetAllSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllIndexes retrieves all of the search indexes for the cluster. @@ -49,15 +51,30 @@ func (sim *SearchIndexManager) GetAllIndexes(opts *GetAllSearchIndexOptions) ([] defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "GET", - Path: "/api/index", - Context: ctx, + Service: gocbcore.ServiceType(SearchService), + Method: "GET", + Path: "/api/index", + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -96,8 +113,9 @@ func (sim *SearchIndexManager) GetAllIndexes(opts *GetAllSearchIndexOptions) ([] // GetSearchIndexOptions is the set of options available to the search indexes GetIndex operation. type GetSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetIndex retrieves a specific search index by name. @@ -111,14 +129,29 @@ func (sim *SearchIndexManager) GetIndex(indexName string, opts *GetSearchIndexOp defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "GET", - Path: fmt.Sprintf("/api/index/%s", indexName), - Context: ctx, + Service: gocbcore.ServiceType(SearchService), + Method: "GET", + Path: fmt.Sprintf("/api/index/%s", indexName), + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -151,8 +184,9 @@ func (sim *SearchIndexManager) GetIndex(indexName string, opts *GetSearchIndexOp // UpsertSearchIndexOptions is the set of options available to the search index manager UpsertIndex operation. type UpsertSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // SearchIndex is used to define a search index. @@ -196,23 +230,37 @@ func (sim *SearchIndexManager) UpsertIndex(indexDefinition SearchIndex, opts *Up defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + b, err := json.Marshal(indexDefinition) if err != nil { return err } req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "PUT", - Path: fmt.Sprintf("/api/index/%s", indexDefinition.Name), - Headers: make(map[string]string), - Context: ctx, - Body: b, + Service: gocbcore.ServiceType(SearchService), + Method: "PUT", + Path: fmt.Sprintf("/api/index/%s", indexDefinition.Name), + Headers: make(map[string]string), + Context: ctx, + Body: b, + RetryStrategy: retryStrategy, } req.Headers["cache-control"] = "no-cache" res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -238,8 +286,9 @@ func (sim *SearchIndexManager) UpsertIndex(indexDefinition SearchIndex, opts *Up // DropSearchIndexOptions is the set of options available to the search index DropIndex operation. type DropSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropIndex removes the search index with the specific name. @@ -257,14 +306,28 @@ func (sim *SearchIndexManager) DropIndex(indexName string, opts *DropSearchIndex defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "DELETE", - Path: fmt.Sprintf("/api/index/%s", indexName), - Context: ctx, + Service: gocbcore.ServiceType(SearchService), + Method: "DELETE", + Path: fmt.Sprintf("/api/index/%s", indexName), + Context: ctx, + RetryStrategy: retryStrategy, } res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -290,8 +353,9 @@ func (sim *SearchIndexManager) DropIndex(indexName string, opts *DropSearchIndex // AnalyzeDocumentOptions is the set of options available to the search index AnalyzeDocument operation. type AnalyzeDocumentOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // AnalyzeDocument returns how a doc is analyzed against a specific index. @@ -309,20 +373,35 @@ func (sim *SearchIndexManager) AnalyzeDocument(indexName string, doc interface{} defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + b, err := json.Marshal(doc) if err != nil { return nil, err } req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "POST", - Path: fmt.Sprintf("/api/index/%s/analyzeDoc", indexName), - Context: ctx, - Body: b, + Service: gocbcore.ServiceType(SearchService), + Method: "POST", + Path: fmt.Sprintf("/api/index/%s/analyzeDoc", indexName), + Context: ctx, + Body: b, + RetryStrategy: retryStrategy, + IsIdempotent: true, } res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -358,8 +437,9 @@ func (sim *SearchIndexManager) AnalyzeDocument(indexName string, doc interface{} // GetIndexedDocumentsCountOptions is the set of options available to the search index GetIndexedDocumentsCount operation. type GetIndexedDocumentsCountOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetIndexedDocumentsCount retrieves the document count for a search index. @@ -377,14 +457,29 @@ func (sim *SearchIndexManager) GetIndexedDocumentsCount(indexName string, opts * defer cancel() } + retryStrategy := sim.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: "GET", - Path: fmt.Sprintf("/api/index/%s/count", indexName), - Context: ctx, + Service: gocbcore.ServiceType(SearchService), + Method: "GET", + Path: fmt.Sprintf("/api/index/%s/count", indexName), + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return 0, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return 0, err } @@ -417,15 +512,29 @@ func (sim *SearchIndexManager) GetIndexedDocumentsCount(indexName string, opts * return count.Count, nil } -func (sim *SearchIndexManager) performControlRequest(ctx context.Context, uri, method string) error { +func (sim *SearchIndexManager) performControlRequest(ctx context.Context, uri, method string, strategy RetryStrategy) error { + retryStrategy := sim.defaultRetryStrategy + if strategy == nil { + retryStrategy = newRetryStrategyWrapper(strategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(SearchService), - Method: method, - Path: uri, - Context: ctx, + Service: gocbcore.ServiceType(SearchService), + Method: method, + Path: uri, + Context: ctx, + RetryStrategy: retryStrategy, } res, err := sim.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -452,8 +561,9 @@ func (sim *SearchIndexManager) performControlRequest(ctx context.Context, uri, m // PauseIngestSearchIndexOptions is the set of options available to the search index PauseIngest operation. type PauseIngestSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // PauseIngest pauses updates and maintenance for an index. @@ -471,13 +581,15 @@ func (sim *SearchIndexManager) PauseIngest(indexName string, opts *PauseIngestSe defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/ingestControl/pause", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/ingestControl/pause", indexName), + "POST", opts.RetryStrategy) } // ResumeIngestSearchIndexOptions is the set of options available to the search index ResumeIngest operation. type ResumeIngestSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // ResumeIngest resumes updates and maintenance for an index. @@ -495,13 +607,15 @@ func (sim *SearchIndexManager) ResumeIngest(indexName string, opts *ResumeIngest defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/ingestControl/resume", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/ingestControl/resume", indexName), + "POST", opts.RetryStrategy) } // AllowQueryingSearchIndexOptions is the set of options available to the search index AllowQuerying operation. type AllowQueryingSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // AllowQuerying allows querying against an index. @@ -519,13 +633,14 @@ func (sim *SearchIndexManager) AllowQuerying(indexName string, opts *AllowQueryi defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/queryControl/allow", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/queryControl/allow", indexName), "POST", opts.RetryStrategy) } // DisallowQueryingSearchIndexOptions is the set of options available to the search index DisallowQuerying operation. type DisallowQueryingSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DisallowQuerying disallows querying against an index. @@ -543,13 +658,15 @@ func (sim *SearchIndexManager) DisallowQuerying(indexName string, opts *AllowQue defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/queryControl/disallow", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/queryControl/disallow", indexName), + "POST", opts.RetryStrategy) } // FreezePlanSearchIndexOptions is the set of options available to the search index FreezePlan operation. type FreezePlanSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // FreezePlan freezes the assignment of index partitions to nodes. @@ -567,13 +684,15 @@ func (sim *SearchIndexManager) FreezePlan(indexName string, opts *AllowQueryingS defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/planFreezeControl/freeze", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/planFreezeControl/freeze", indexName), + "POST", opts.RetryStrategy) } // UnfreezePlanSearchIndexOptions is the set of options available to the search index UnfreezePlan operation. type UnfreezePlanSearchIndexOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // UnfreezePlan unfreezes the assignment of index partitions to nodes. @@ -591,5 +710,6 @@ func (sim *SearchIndexManager) UnfreezePlan(indexName string, opts *AllowQueryin defer cancel() } - return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/planFreezeControl/unfreeze", indexName), "POST") + return sim.performControlRequest(ctx, fmt.Sprintf("/api/index/%s/planFreezeControl/unfreeze", indexName), + "POST", opts.RetryStrategy) } diff --git a/cluster_searchquery.go b/cluster_searchquery.go index 188c654a..eeecb5b7 100644 --- a/cluster_searchquery.go +++ b/cluster_searchquery.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "net/url" "time" @@ -499,23 +500,12 @@ func (c *Cluster) searchQuery(ctx context.Context, qIndexName string, q interfac opts.Serializer = &DefaultJSONSerializer{} } - var retries uint - var res *SearchResult - for { - retries++ - res, err = c.executeSearchQuery(ctx, queryData, qIndexName, provider, cancel, opts.Serializer) - if err == nil { - break - } - - if !IsRetryableError(err) || c.sb.SearchRetryBehavior == nil || !c.sb.SearchRetryBehavior.CanRetry(retries) { - break - } - - time.Sleep(c.sb.SearchRetryBehavior.NextInterval(retries)) - + wrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + wrapper = newRetryStrategyWrapper(opts.RetryStrategy) } + res, err := c.executeSearchQuery(ctx, queryData, qIndexName, provider, cancel, opts.Serializer, wrapper) if err != nil { // only cancel on error, if we cancel when things have gone to plan then we'll prematurely close the stream if cancel != nil { @@ -528,7 +518,7 @@ func (c *Cluster) searchQuery(ctx context.Context, qIndexName string, q interfac } func (c *Cluster) executeSearchQuery(ctx context.Context, query jsonx.DelayedObject, - qIndexName string, provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer) (*SearchResult, error) { + qIndexName string, provider httpProvider, cancel context.CancelFunc, serializer JSONSerializer, wrapper *retryStrategyWrapper) (*SearchResult, error) { qBytes, err := json.Marshal(query) if err != nil { @@ -536,119 +526,146 @@ func (c *Cluster) executeSearchQuery(ctx context.Context, query jsonx.DelayedObj } req := &gocbcore.HttpRequest{ - Service: gocbcore.FtsService, - Path: fmt.Sprintf("/api/index/%s/query", qIndexName), - Method: "POST", - Context: ctx, - Body: qBytes, + Service: gocbcore.FtsService, + Path: fmt.Sprintf("/api/index/%s/query", qIndexName), + Method: "POST", + Context: ctx, + Body: qBytes, + IsIdempotent: true, + RetryStrategy: wrapper, } - resp, err := provider.DoHttpRequest(req) - if err != nil { - if err == gocbcore.ErrNoFtsService { - return nil, serviceNotAvailableError{message: gocbcore.ErrNoFtsService.Error()} - } + for { + resp, err := provider.DoHttpRequest(req) + if err != nil { + if err == gocbcore.ErrNoFtsService { + return nil, serviceNotAvailableError{message: gocbcore.ErrNoFtsService.Error()} + } - // as we're effectively manually timing out the request using cancellation we need - // to check if the original context has timed out as err itself will only show as canceled - if ctx.Err() == context.DeadlineExceeded { - return nil, timeoutError{} - } - return nil, errors.Wrap(err, "could not complete search http request") - } + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } - epInfo, err := url.Parse(resp.Endpoint) - if err != nil { - logWarnf("Failed to parse N1QL source address") - epInfo = &url.URL{ - Host: "", + return nil, errors.Wrap(err, "could not complete search http request") } - } - switch resp.StatusCode { - case 400: - // This goes against the FTS RFC but makes a better experience in Go - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(resp.Body) + epInfo, err := url.Parse(resp.Endpoint) if err != nil { - return nil, err - } - respErrs := []string{buf.String()} - var errs []SearchError - for _, err := range respErrs { - errs = append(errs, searchError{ - message: err, - }) - } - return nil, searchMultiError{ - errors: errs, - endpoint: epInfo.Host, - httpStatus: resp.StatusCode, + logWarnf("Failed to parse N1QL source address") + epInfo = &url.URL{ + Host: "", + } } - case 401: - // This goes against the FTS RFC but makes a better experience in Go - return nil, searchMultiError{ - errors: []SearchError{ - searchError{ - message: "The requested consistency level could not be satisfied before the timeout was reached", + + switch resp.StatusCode { + case 400: + // This goes against the FTS RFC but makes a better experience in Go + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + respErrs := []string{buf.String()} + var errs []SearchError + for _, err := range respErrs { + errs = append(errs, searchError{ + message: err, + }) + } + return nil, searchMultiError{ + errors: errs, + endpoint: epInfo.Host, + httpStatus: resp.StatusCode, + } + case 401: + // This goes against the FTS RFC but makes a better experience in Go + return nil, searchMultiError{ + errors: []SearchError{ + searchError{ + message: "The requested consistency level could not be satisfied before the timeout was reached", + }, }, - }, - endpoint: epInfo.Host, - httpStatus: resp.StatusCode, + endpoint: epInfo.Host, + httpStatus: resp.StatusCode, + } + case 429: + shouldRetry, retryErr := shouldRetryHTTPRequest(ctx, req, gocbcore.ServiceResponseCodeIndicatedRetryReason, + wrapper, provider) + if shouldRetry { + continue + } + + if retryErr != nil { + return nil, retryErr + } + + // Drop out here and we'll create an error below. } - } - if resp.StatusCode != 200 { - err = searchMultiError{ - errors: []SearchError{ - searchError{ - message: "An unknown error occurred", + if resp.StatusCode != 200 { + errMsg := "An unknown error occurred" + errBytes, bodyErr := ioutil.ReadAll(resp.Body) + if bodyErr == nil { + errMsg = string(errBytes) + } else { + logDebugf("Failed to ready message from body (%s)", bodyErr.Error()) + } + + err = searchMultiError{ + errors: []SearchError{ + searchError{ + message: errMsg, + }, }, + endpoint: epInfo.Host, + httpStatus: resp.StatusCode, + } + + return nil, err + } + + queryResults := &SearchResult{ + metadata: SearchMetadata{ + sourceAddr: epInfo.Host, }, - endpoint: epInfo.Host, httpStatus: resp.StatusCode, + serializer: serializer, } - return nil, err - } - - queryResults := &SearchResult{ - metadata: SearchMetadata{ - sourceAddr: epInfo.Host, - }, - httpStatus: resp.StatusCode, - serializer: serializer, - } + streamResult, err := newStreamingResults(resp.Body, queryResults.readAttribute) + if err != nil { + return nil, err + } - streamResult, err := newStreamingResults(resp.Body, queryResults.readAttribute) - if err != nil { - return nil, err - } + err = streamResult.readAttributes() + if err != nil { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close socket (%s)", bodyErr.Error()) + } - err = streamResult.readAttributes() - if err != nil { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close socket (%s)", bodyErr.Error()) + return nil, err } - return nil, err - } - queryResults.streamResult = streamResult + queryResults.streamResult = streamResult - if streamResult.HasRows() { - queryResults.cancel = cancel - queryResults.ctx = ctx - } else { - bodyErr := streamResult.Close() - if bodyErr != nil { - logDebugf("Failed to close response body, %s", bodyErr.Error()) - } + if streamResult.HasRows() { + queryResults.cancel = cancel + queryResults.ctx = ctx + } else { + bodyErr := streamResult.Close() + if bodyErr != nil { + logDebugf("Failed to close response body, %s", bodyErr.Error()) + } + + // We've already handled the retry case above. - // There are no rows and there are errors so fast fail - if queryResults.err != nil { return nil, queryResults.err } + return queryResults, nil } - return queryResults, nil } diff --git a/cluster_searchquery_test.go b/cluster_searchquery_test.go index f01ddbee..ffc36074 100644 --- a/cluster_searchquery_test.go +++ b/cluster_searchquery_test.go @@ -2,6 +2,7 @@ package gocb import ( "bytes" + "context" "encoding/json" "fmt" "testing" @@ -227,9 +228,13 @@ func TestSearchQueryRetries(t *testing.T) { doHTTP := func(req *gocbcore.HttpRequest) (*gocbcore.HttpResponse, error) { retries++ + if retries == 3 { + return nil, context.DeadlineExceeded + } + return &gocbcore.HttpResponse{ Endpoint: "http://localhost:8093", - StatusCode: 419, + StatusCode: 429, }, nil } @@ -237,16 +242,15 @@ func TestSearchQueryRetries(t *testing.T) { doFn: doHTTP, } - cluster := testGetClusterForHTTP(provider, timeout, 0, 0) - cluster.sb.SearchRetryBehavior = standardDelayRetryBehavior(3, 1, 100*time.Millisecond, linearDelayFunction) + cluster := testGetClusterForHTTP(provider, 0, 0, timeout) _, err := cluster.SearchQuery("test", NewMatchQuery("test"), nil) - if err == nil { - t.Fatal("Expected query execution to error") + if !IsTimeoutError(err) { + t.Fatalf("Expected query execution to be timeout error, was %v", err) } if retries != 3 { - t.Fatalf("Expected query to be retried 3 time but ws retried %d times", retries) + t.Fatalf("Expected query to be retried 3 time but was retried %d times", retries) } } diff --git a/cluster_usermgr.go b/cluster_usermgr.go index a3aa4dcb..c3f751e3 100644 --- a/cluster_usermgr.go +++ b/cluster_usermgr.go @@ -15,8 +15,9 @@ import ( // UserManager provides methods for performing Couchbase user management. // Volatile: This API is subject to change at any time. type UserManager struct { - httpClient httpProvider - globalTimeout time.Duration + httpClient httpProvider + globalTimeout time.Duration + defaultRetryStrategy *retryStrategyWrapper } // Role represents a specific permission. @@ -153,8 +154,9 @@ func transformUserMetadataJson(userData *userMetadataJson) UserAndMetadata { // GetAllUsersOptions is the set of options available to the user manager GetAll operation. type GetAllUsersOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy DomainName string } @@ -174,15 +176,30 @@ func (um *UserManager) GetAllUsers(opts *GetAllUsersOptions) ([]UserAndMetadata, defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "GET", - Path: fmt.Sprintf("/settings/rbac/users/%s", opts.DomainName), - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "GET", + Path: fmt.Sprintf("/settings/rbac/users/%s", opts.DomainName), + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -216,8 +233,9 @@ func (um *UserManager) GetAllUsers(opts *GetAllUsersOptions) ([]UserAndMetadata, // GetUserOptions is the set of options available to the user manager Get operation. type GetUserOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy DomainName string } @@ -237,15 +255,30 @@ func (um *UserManager) GetUser(name string, opts *GetUserOptions) (*UserAndMetad defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "GET", - Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, name), - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "GET", + Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, name), + Context: ctx, + IsIdempotent: true, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -274,8 +307,9 @@ func (um *UserManager) GetUser(name string, opts *GetUserOptions) (*UserAndMetad // UpsertUserOptions is the set of options available to the user manager Upsert operation. type UpsertUserOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy DomainName string } @@ -295,6 +329,11 @@ func (um *UserManager) UpsertUser(user User, opts *UpsertUserOptions) error { defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + var reqRoleStrs []string for _, roleData := range user.Roles { reqRoleStrs = append(reqRoleStrs, fmt.Sprintf("%s[%s]", roleData.Name, roleData.Bucket)) @@ -311,16 +350,25 @@ func (um *UserManager) UpsertUser(user User, opts *UpsertUserOptions) error { reqForm.Add("roles", strings.Join(reqRoleStrs, ",")) req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "PUT", - Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, user.Username), - Body: []byte(reqForm.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "PUT", + Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, user.Username), + Body: []byte(reqForm.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -341,8 +389,9 @@ func (um *UserManager) UpsertUser(user User, opts *UpsertUserOptions) error { // DropUserOptions is the set of options available to the user manager Drop operation. type DropUserOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy DomainName string } @@ -362,15 +411,29 @@ func (um *UserManager) DropUser(name string, opts *DropUserOptions) error { defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "DELETE", - Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, name), - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "DELETE", + Path: fmt.Sprintf("/settings/rbac/users/%s/%s", opts.DomainName, name), + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -391,8 +454,9 @@ func (um *UserManager) DropUser(name string, opts *DropUserOptions) error { // GetRolesOptions is the set of options available to the user manager GetRoles operation. type GetRolesOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetRoles lists the roles supported by the cluster. @@ -406,15 +470,30 @@ func (um *UserManager) GetRoles(opts *GetRolesOptions) ([]RoleAndDescription, er defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "GET", - Path: "/settings/rbac/roles", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "GET", + Path: "/settings/rbac/roles", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -456,8 +535,9 @@ func (um *UserManager) GetRoles(opts *GetRolesOptions) ([]RoleAndDescription, er // GetGroupOptions is the set of options available to the group manager Get operation. type GetGroupOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetGroup fetches a single group from the server. @@ -474,15 +554,30 @@ func (um *UserManager) GetGroup(groupName string, opts *GetGroupOptions) (*Group defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "GET", - Path: fmt.Sprintf("/settings/rbac/groups/%s", groupName), - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "GET", + Path: fmt.Sprintf("/settings/rbac/groups/%s", groupName), + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -510,8 +605,9 @@ func (um *UserManager) GetGroup(groupName string, opts *GetGroupOptions) (*Group // GetAllGroupsOptions is the set of options available to the group manager GetAll operation. type GetAllGroupsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // GetAllGroups fetches all groups from the server. @@ -525,15 +621,30 @@ func (um *UserManager) GetAllGroups(opts *GetAllGroupsOptions) ([]Group, error) defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "GET", - Path: "/settings/rbac/groups", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "GET", + Path: "/settings/rbac/groups", + Context: ctx, + RetryStrategy: retryStrategy, + IsIdempotent: true, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return nil, timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return nil, err } @@ -561,8 +672,9 @@ func (um *UserManager) GetAllGroups(opts *GetAllGroupsOptions) ([]Group, error) // UpsertGroupOptions is the set of options available to the group manager Upsert operation. type UpsertGroupOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // UpsertGroup creates, or updates, a group on the server. @@ -579,6 +691,11 @@ func (um *UserManager) UpsertGroup(group Group, opts *UpsertGroupOptions) error defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + var reqRoleStrs []string for _, roleData := range group.Roles { if roleData.Bucket == "" { @@ -594,16 +711,25 @@ func (um *UserManager) UpsertGroup(group Group, opts *UpsertGroupOptions) error reqForm.Add("roles", strings.Join(reqRoleStrs, ",")) req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "PUT", - Path: fmt.Sprintf("/settings/rbac/groups/%s", group.Name), - Body: []byte(reqForm.Encode()), - ContentType: "application/x-www-form-urlencoded", - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "PUT", + Path: fmt.Sprintf("/settings/rbac/groups/%s", group.Name), + Body: []byte(reqForm.Encode()), + ContentType: "application/x-www-form-urlencoded", + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } @@ -624,8 +750,9 @@ func (um *UserManager) UpsertGroup(group Group, opts *UpsertGroupOptions) error // DropGroupOptions is the set of options available to the group manager Drop operation. type DropGroupOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // DropGroup removes a group from the server. @@ -643,15 +770,29 @@ func (um *UserManager) DropGroup(groupName string, opts *DropGroupOptions) error defer cancel() } + retryStrategy := um.defaultRetryStrategy + if opts.RetryStrategy == nil { + retryStrategy = newRetryStrategyWrapper(opts.RetryStrategy) + } + req := &gocbcore.HttpRequest{ - Service: gocbcore.ServiceType(MgmtService), - Method: "DELETE", - Path: fmt.Sprintf("/settings/rbac/groups/%s", groupName), - Context: ctx, + Service: gocbcore.ServiceType(MgmtService), + Method: "DELETE", + Path: fmt.Sprintf("/settings/rbac/groups/%s", groupName), + Context: ctx, + RetryStrategy: retryStrategy, } resp, err := um.httpClient.DoHttpRequest(req) if err != nil { + if err == context.DeadlineExceeded { + return timeoutError{ + operationID: req.UniqueId, + retryReasons: req.RetryReasons(), + retryAttempts: req.RetryAttempts(), + } + } + return err } diff --git a/collection_binary_crud.go b/collection_binary_crud.go index b50116d6..8d3a4e00 100644 --- a/collection_binary_crud.go +++ b/collection_binary_crud.go @@ -20,6 +20,7 @@ type AppendOptions struct { PersistTo uint ReplicateTo uint Cas Cas + RetryStrategy RetryStrategy } // Append appends a byte value to a document. @@ -66,6 +67,11 @@ func (c *BinaryCollection) append(ctx context.Context, id string, val []byte, op return nil, err } + retryWrapper := c.collection.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.collection.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -82,6 +88,7 @@ func (c *BinaryCollection) append(ctx context.Context, id string, val []byte, op DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, Cas: gocbcore.Cas(opts.Cas), + RetryStrategy: retryWrapper, }, func(res *gocbcore.AdjoinResult, err error) { if err != nil { errOut = err @@ -117,6 +124,7 @@ type PrependOptions struct { PersistTo uint ReplicateTo uint Cas Cas + RetryStrategy RetryStrategy } // Prepend prepends a byte value to a document. @@ -163,6 +171,11 @@ func (c *BinaryCollection) prepend(ctx context.Context, id string, val []byte, o return nil, err } + retryWrapper := c.collection.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.collection.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -179,6 +192,7 @@ func (c *BinaryCollection) prepend(ctx context.Context, id string, val []byte, o DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, Cas: gocbcore.Cas(opts.Cas), + RetryStrategy: retryWrapper, }, func(res *gocbcore.AdjoinResult, err error) { if err != nil { errOut = err @@ -222,6 +236,7 @@ type CounterOptions struct { PersistTo uint ReplicateTo uint Cas Cas + RetryStrategy RetryStrategy } // Increment performs an atomic addition for an integer document. Passing a @@ -265,6 +280,11 @@ func (c *BinaryCollection) increment(ctx context.Context, id string, opts Counte realInitial = uint64(opts.Initial) } + retryWrapper := c.collection.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + agent, err := c.collection.getKvProvider() if err != nil { return nil, err @@ -288,6 +308,7 @@ func (c *BinaryCollection) increment(ctx context.Context, id string, opts Counte DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, Cas: gocbcore.Cas(opts.Cas), + RetryStrategy: retryWrapper, }, func(res *gocbcore.CounterResult, err error) { if err != nil { errOut = err @@ -372,6 +393,11 @@ func (c *BinaryCollection) decrement(ctx context.Context, id string, opts Counte realInitial = uint64(opts.Initial) } + retryWrapper := c.collection.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.collection.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -390,6 +416,7 @@ func (c *BinaryCollection) decrement(ctx context.Context, id string, opts Counte DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, Cas: gocbcore.Cas(opts.Cas), + RetryStrategy: retryWrapper, }, func(res *gocbcore.CounterResult, err error) { if err != nil { errOut = err diff --git a/collection_bulk.go b/collection_bulk.go index e44085ce..effd9861 100644 --- a/collection_bulk.go +++ b/collection_bulk.go @@ -24,7 +24,7 @@ func (op *bulkOp) cancel() bool { // You can create a bulk operation by instantiating one of the implementations of BulkOp, // such as GetOp, UpsertOp, ReplaceOp, and more. type BulkOp interface { - execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) + execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) markError(err error) cancel() bool } @@ -36,7 +36,8 @@ type BulkOpOptions struct { // Transcoder is used to encode values for operations that perform mutations and to decode values for // operations that fetch values. It does not apply to all BulkOp operations. - Transcoder Transcoder + Transcoder Transcoder + RetryStrategy RetryStrategy } // Do execute one or more `BulkOp` items in parallel. @@ -61,6 +62,11 @@ func (c *Collection) Do(ops []BulkOp, opts *BulkOpOptions) error { } defer cancel() + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + if opts.Transcoder == nil { opts.Transcoder = c.sb.Transcoder } @@ -75,7 +81,7 @@ func (c *Collection) Do(ops []BulkOp, opts *BulkOpOptions) error { // individual op handlers when they dispatch their signal). signal := make(chan BulkOp, len(ops)) for _, item := range ops { - item.execute(c, agent, opts.Transcoder, signal) + item.execute(c, agent, opts.Transcoder, signal, retryWrapper) } for range ops { select { @@ -114,11 +120,12 @@ func (item *GetOp) markError(err error) { item.Err = err } -func (item *GetOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *GetOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.GetEx(gocbcore.GetOptions{ Key: []byte(item.ID), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, false) if item.Err == nil { @@ -155,12 +162,13 @@ func (item *GetAndTouchOp) markError(err error) { item.Err = err } -func (item *GetAndTouchOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *GetAndTouchOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.GetAndTouchEx(gocbcore.GetAndTouchOptions{ Key: []byte(item.ID), Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetAndTouchResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, false) if item.Err == nil { @@ -197,12 +205,13 @@ func (item *TouchOp) markError(err error) { item.Err = err } -func (item *TouchOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *TouchOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.TouchEx(gocbcore.TouchOptions{ Key: []byte(item.ID), Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.TouchResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, false) if item.Err == nil { @@ -244,12 +253,13 @@ func (item *RemoveOp) markError(err error) { item.Err = err } -func (item *RemoveOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *RemoveOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.DeleteEx(gocbcore.DeleteOptions{ Key: []byte(item.ID), Cas: gocbcore.Cas(item.Cas), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.DeleteResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, false) if item.Err == nil { @@ -293,7 +303,7 @@ func (item *UpsertOp) markError(err error) { item.Err = err } -func (item *UpsertOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *UpsertOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { bytes, flags, err := transcoder.Encode(item.Value) if err != nil { item.Err = err @@ -308,6 +318,7 @@ func (item *UpsertOp) execute(c *Collection, provider kvProvider, transcoder Tra Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, false) if item.Err == nil { @@ -350,7 +361,7 @@ func (item *InsertOp) markError(err error) { item.Err = err } -func (item *InsertOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *InsertOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { bytes, flags, err := transcoder.Encode(item.Value) if err != nil { item.Err = err @@ -365,6 +376,7 @@ func (item *InsertOp) execute(c *Collection, provider kvProvider, transcoder Tra Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { @@ -408,7 +420,7 @@ func (item *ReplaceOp) markError(err error) { item.Err = err } -func (item *ReplaceOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *ReplaceOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { bytes, flags, err := transcoder.Encode(item.Value) if err != nil { item.Err = err @@ -424,6 +436,7 @@ func (item *ReplaceOp) execute(c *Collection, provider kvProvider, transcoder Tr Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { @@ -465,12 +478,13 @@ func (item *AppendOp) markError(err error) { item.Err = err } -func (item *AppendOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *AppendOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.AppendEx(gocbcore.AdjoinOptions{ Key: []byte(item.ID), Value: []byte(item.Value), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.AdjoinResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { @@ -512,12 +526,13 @@ func (item *PrependOp) markError(err error) { item.Err = err } -func (item *PrependOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *PrependOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { op, err := provider.PrependEx(gocbcore.AdjoinOptions{ Key: []byte(item.ID), Value: []byte(item.Value), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.AdjoinResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { @@ -562,7 +577,7 @@ func (item *IncrementOp) markError(err error) { item.Err = err } -func (item *IncrementOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *IncrementOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { realInitial := uint64(0xFFFFFFFFFFFFFFFF) if item.Initial > 0 { realInitial = uint64(item.Initial) @@ -575,6 +590,7 @@ func (item *IncrementOp) execute(c *Collection, provider kvProvider, transcoder Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.CounterResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { @@ -622,7 +638,7 @@ func (item *DecrementOp) markError(err error) { item.Err = err } -func (item *DecrementOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp) { +func (item *DecrementOp) execute(c *Collection, provider kvProvider, transcoder Transcoder, signal chan BulkOp, retryWrapper *retryStrategyWrapper) { realInitial := uint64(0xFFFFFFFFFFFFFFFF) if item.Initial > 0 { realInitial = uint64(item.Initial) @@ -635,6 +651,7 @@ func (item *DecrementOp) execute(c *Collection, provider kvProvider, transcoder Expiry: item.Expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.CounterResult, err error) { item.Err = maybeEnhanceKVErr(err, item.ID, true) if item.Err == nil { diff --git a/collection_crud.go b/collection_crud.go index 79632e5a..2cd21f7f 100644 --- a/collection_crud.go +++ b/collection_crud.go @@ -27,7 +27,7 @@ type kvProvider interface { DecrementEx(opts gocbcore.CounterOptions, cb gocbcore.CounterExCallback) (gocbcore.PendingOp, error) AppendEx(opts gocbcore.AdjoinOptions, cb gocbcore.AdjoinExCallback) (gocbcore.PendingOp, error) PrependEx(opts gocbcore.AdjoinOptions, cb gocbcore.AdjoinExCallback) (gocbcore.PendingOp, error) - PingKvEx(opts gocbcore.PingKvOptions, cb gocbcore.PingKvExCallback) (gocbcore.PendingOp, error) + PingKvEx(opts gocbcore.PingKvOptions, cb gocbcore.PingKvExCallback) (gocbcore.CancellablePendingOp, error) NumReplicas() int } @@ -72,7 +72,11 @@ func (ctrl *opManager) wait(op gocbcore.PendingOp, err error) (errOut error) { if op.Cancel() { ctxErr := ctrl.ctx.Err() if ctxErr == context.DeadlineExceeded { - errOut = timeoutError{} + err := timeoutError{} + err.operationID = op.Identifier() + err.retryAttempts = op.RetryAttempts() + err.retryReasons = op.RetryReasons() + errOut = err } else { errOut = ctxErr } @@ -131,6 +135,7 @@ type UpsertOptions struct { ReplicateTo uint DurabilityLevel DurabilityLevel Transcoder Transcoder + RetryStrategy RetryStrategy } // InsertOptions are options that can be applied to an Insert operation. @@ -143,6 +148,7 @@ type InsertOptions struct { ReplicateTo uint DurabilityLevel DurabilityLevel Transcoder Transcoder + RetryStrategy RetryStrategy } // Insert creates a new document in the Collection. @@ -200,6 +206,11 @@ func (c *Collection) insert(ctx context.Context, id string, val interface{}, opt return } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -217,6 +228,7 @@ func (c *Collection) insert(ctx context.Context, id string, val interface{}, opt ScopeName: c.scopeName(), DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, true) @@ -293,6 +305,11 @@ func (c *Collection) upsert(ctx context.Context, id string, val interface{}, opt transcoder = c.sb.Transcoder } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + bytes, flags, err := transcoder.Encode(val) if err != nil { errOut = err @@ -316,6 +333,7 @@ func (c *Collection) upsert(ctx context.Context, id string, val interface{}, opt ScopeName: c.scopeName(), DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -353,6 +371,7 @@ type ReplaceOptions struct { ReplicateTo uint DurabilityLevel DurabilityLevel Transcoder Transcoder + RetryStrategy RetryStrategy } // Replace updates a document in the collection. @@ -403,6 +422,11 @@ func (c *Collection) replace(ctx context.Context, id string, val interface{}, op transcoder = c.sb.Transcoder } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + bytes, flags, err := transcoder.Encode(val) if err != nil { errOut = err @@ -427,6 +451,7 @@ func (c *Collection) replace(ctx context.Context, id string, val interface{}, op ScopeName: c.scopeName(), DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, + RetryStrategy: retryWrapper, }, func(res *gocbcore.StoreResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -462,8 +487,9 @@ type GetOptions struct { // Project causes the Get operation to only fetch the fields indicated // by the paths. The result of the operation is then treated as a // standard GetResult. - Project []string - Transcoder Transcoder + Project []string + Transcoder Transcoder + RetryStrategy RetryStrategy } // Get performs a fetch operation against the collection. This can take 3 paths, a standard full document @@ -557,11 +583,17 @@ func (c *Collection) get(ctx context.Context, id string, opts *GetOptions) (docO return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.GetEx(gocbcore.GetOptions{ Key: []byte(id), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -580,7 +612,6 @@ func (c *Collection) get(ctx context.Context, id string, opts *GetOptions) (docO docOut = doc } - ctrl.resolve() })) if err != nil { @@ -592,8 +623,9 @@ func (c *Collection) get(ctx context.Context, id string, opts *GetOptions) (docO // ExistsOptions are the options available to the Exists command. type ExistsOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // Exists checks if a document exists for the given id. @@ -621,12 +653,18 @@ func (c *Collection) exists(ctx context.Context, id string, opts ExistsOptions) return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.ObserveEx(gocbcore.ObserveOptions{ Key: []byte(id), ReplicaIdx: 0, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.ObserveResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -655,9 +693,10 @@ func (c *Collection) exists(ctx context.Context, id string, opts ExistsOptions) // GetAnyReplicaOptions are the options available to the GetAnyReplica command. type GetAnyReplicaOptions struct { - Timeout time.Duration - Context context.Context - Transcoder Transcoder + Timeout time.Duration + Context context.Context + Transcoder Transcoder + RetryStrategy RetryStrategy } // GetAnyReplica returns the value of a particular document from a replica server. @@ -690,11 +729,17 @@ func (c *Collection) getAnyReplica(ctx context.Context, id string, opts.Transcoder = c.sb.Transcoder } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.GetAnyReplicaEx(gocbcore.GetAnyReplicaOptions{ Key: []byte(id), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetReplicaResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -726,9 +771,10 @@ func (c *Collection) getAnyReplica(ctx context.Context, id string, // GetAllReplicaOptions are the options available to the GetAllReplicas command. type GetAllReplicaOptions struct { - Timeout time.Duration - Context context.Context - Transcoder Transcoder + Timeout time.Duration + Context context.Context + Transcoder Transcoder + RetryStrategy RetryStrategy } // GetAllReplicas returns the value of a particular document from all replica servers. This will return an iterable @@ -744,6 +790,11 @@ func (c *Collection) GetAllReplicas(id string, opts *GetAllReplicaOptions) (docO return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + if opts.Transcoder == nil { opts.Transcoder = c.sb.Transcoder } @@ -754,6 +805,7 @@ func (c *Collection) GetAllReplicas(id string, opts *GetAllReplicaOptions) (docO Key: []byte(id), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, transcoder: opts.Transcoder, provider: agent, @@ -770,6 +822,7 @@ type RemoveOptions struct { PersistTo uint ReplicateTo uint DurabilityLevel DurabilityLevel + RetryStrategy RetryStrategy } // Remove removes a document from the collection. @@ -815,6 +868,11 @@ func (c *Collection) remove(ctx context.Context, id string, opts RemoveOptions) return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -830,6 +888,7 @@ func (c *Collection) remove(ctx context.Context, id string, opts RemoveOptions) ScopeName: c.scopeName(), DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, + RetryStrategy: retryWrapper, }, func(res *gocbcore.DeleteResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -859,9 +918,10 @@ func (c *Collection) remove(ctx context.Context, id string, opts RemoveOptions) // GetAndTouchOptions are the options available to the GetAndTouch operation. type GetAndTouchOptions struct { - Timeout time.Duration - Context context.Context - Transcoder Transcoder + Timeout time.Duration + Context context.Context + Transcoder Transcoder + RetryStrategy RetryStrategy } // GetAndTouch retrieves a document and simultaneously updates its expiry time. @@ -893,12 +953,18 @@ func (c *Collection) getAndTouch(ctx context.Context, id string, expiry uint32, opts.Transcoder = c.sb.Transcoder } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.GetAndTouchEx(gocbcore.GetAndTouchOptions{ Key: []byte(id), Expiry: expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetAndTouchResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -929,9 +995,10 @@ func (c *Collection) getAndTouch(ctx context.Context, id string, expiry uint32, // GetAndLockOptions are the options available to the GetAndLock operation. type GetAndLockOptions struct { - Timeout time.Duration - Context context.Context - Transcoder Transcoder + Timeout time.Duration + Context context.Context + Transcoder Transcoder + RetryStrategy RetryStrategy } // GetAndLock locks a document for a period of time, providing exclusive RW access to it. @@ -964,12 +1031,18 @@ func (c *Collection) getAndLock(ctx context.Context, id string, lockTime uint32, opts.Transcoder = c.sb.Transcoder } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.GetAndLockEx(gocbcore.GetAndLockOptions{ Key: []byte(id), LockTime: lockTime, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.GetAndLockResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -1000,8 +1073,9 @@ func (c *Collection) getAndLock(ctx context.Context, id string, lockTime uint32, // UnlockOptions are the options available to the GetAndLock operation. type UnlockOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // Unlock unlocks a document which was locked with GetAndLock. @@ -1029,12 +1103,18 @@ func (c *Collection) unlock(ctx context.Context, id string, cas Cas, opts Unlock return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.UnlockEx(gocbcore.UnlockOptions{ Key: []byte(id), Cas: gocbcore.Cas(cas), CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.UnlockResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) @@ -1064,8 +1144,9 @@ func (c *Collection) unlock(ctx context.Context, id string, cas Cas, opts Unlock // TouchOptions are the options available to the Touch operation. type TouchOptions struct { - Timeout time.Duration - Context context.Context + Timeout time.Duration + Context context.Context + RetryStrategy RetryStrategy } // Touch touches a document, specifying a new expiry time for it. @@ -1093,12 +1174,18 @@ func (c *Collection) touch(ctx context.Context, id string, expiry uint32, opts T return nil, err } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.TouchEx(gocbcore.TouchOptions{ Key: []byte(id), Expiry: expiry, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.TouchResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, false) diff --git a/collection_crud_test.go b/collection_crud_test.go index 319ac2f6..a255b248 100644 --- a/collection_crud_test.go +++ b/collection_crud_test.go @@ -708,6 +708,59 @@ func TestUpsertGetRemove(t *testing.T) { } } +type upsertRetriesStrategy struct { + retries int +} + +func (rts *upsertRetriesStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction { + rts.retries++ + return &WithDurationRetryAction{100 * time.Millisecond} +} + +func TestUpsertRetries(t *testing.T) { + if testing.Short() { + t.Log("Skipping test in short mode") + return + } + + var doc testBeerDocument + err := loadJSONTestDataset("beer_sample_single", &doc) + if err != nil { + t.Fatalf("Could not read test dataset: %v", err) + } + + mutRes, err := globalCollection.Upsert("getRetryDoc", doc, nil) + if err != nil { + t.Fatalf("Upsert failed, error was %v", err) + } + + if mutRes.Cas() == 0 { + t.Fatalf("Upsert CAS was 0") + } + + _, err = globalCollection.GetAndLock("getRetryDoc", 1*time.Second, nil) + if err != nil { + t.Fatalf("GetAndLock failed, error was %v", err) + } + + retryStrategy := &upsertRetriesStrategy{} + mutRes, err = globalCollection.Upsert("getRetryDoc", doc, &UpsertOptions{ + Timeout: 2100 * time.Millisecond, // Timeout has to be long due to how the server handles unlocking. + RetryStrategy: retryStrategy, + }) + if err != nil { + t.Fatalf("Upsert failed, error was %v", err) + } + + if mutRes.Cas() == 0 { + t.Fatalf("Upsert CAS was 0") + } + + if retryStrategy.retries <= 1 { + t.Fatalf("Expected retries to be > 1") + } +} + func TestRemoveWithCas(t *testing.T) { var doc testBeerDocument err := loadJSONTestDataset("beer_sample_single", &doc) @@ -909,12 +962,14 @@ func TestGetAndLock(t *testing.T) { t.Fatalf("Expected resulting doc to be %v but was %v", doc, lockedDocContent) } - mutRes, err = globalCollection.Upsert("getAndLock", doc, nil) + mutRes, err = globalCollection.Upsert("getAndLock", doc, &UpsertOptions{ + RetryStrategy: NewFailFastRetryStrategy(), + }) if err == nil { t.Fatalf("Expected error but was nil") } - if !IsKeyExistsError(err) { + if !IsKeyLockedError(err) { t.Fatalf("Expected error to be KeyExistsError but is %s", reflect.TypeOf(err).String()) } @@ -992,7 +1047,7 @@ func TestUnlockInvalidCas(t *testing.T) { t.Fatalf("Upsert CAS was 0") } - lockedDoc, err := globalCollection.GetAndLock("unlockInvalidCas", 1, nil) + lockedDoc, err := globalCollection.GetAndLock("unlockInvalidCas", 2, nil) if err != nil { t.Fatalf("Get failed, error was %v", err) } @@ -1007,13 +1062,41 @@ func TestUnlockInvalidCas(t *testing.T) { t.Fatalf("Expected resulting doc to be %v but was %v", doc, lockedDocContent) } - _, err = globalCollection.Unlock("unlockInvalidCas", lockedDoc.Cas()+1, nil) + _, err = globalCollection.Unlock("unlockInvalidCas", lockedDoc.Cas()+1, &UnlockOptions{ + RetryStrategy: NewFailFastRetryStrategy(), + }) if err == nil { t.Fatalf("Unlock should have failed") } - if !IsTemporaryFailureError(err) { - t.Fatalf("Expected error to be TempFailError but was %s", reflect.TypeOf(err).String()) + // The server and the mock do not agree on the error for locked documents. + if !IsKeyLockedError(err) && !IsTemporaryFailureError(err) { + t.Fatalf("Expected error to be TempFailError or IsKeyLockedError but was %s", reflect.TypeOf(err).String()) + } + + _, err = globalCollection.Unlock("unlockInvalidCas", lockedDoc.Cas()+1, &UnlockOptions{ + RetryStrategy: NewBestEffortRetryStrategy(nil), + Timeout: 10 * time.Millisecond, + }) + if err == nil { + t.Fatalf("Unlock should have failed") + } + + if !IsTimeoutError(err) { + t.Fatalf("Expected error to be TimeoutError but was %s", reflect.TypeOf(err).String()) + } + + tErr, ok := err.(TimeoutErrorWithDetail) + if !ok { + t.Fatalf("Expected error to be TimeoutErrorWithDetail but was %+v", err) + } + + if len(tErr.RetryReasons()) != 1 { + t.Fatalf("Expected 1 retry reason but was %v", tErr.RetryReasons()) + } + + if tErr.OperationID() == "" { + t.Fatalf("Expected OperationID to be not empty") } } @@ -1048,13 +1131,16 @@ func TestDoubleLockFail(t *testing.T) { t.Fatalf("Expected resulting doc to be %v but was %v", doc, lockedDocContent) } - _, err = globalCollection.GetAndLock("doubleLock", 1, nil) + _, err = globalCollection.GetAndLock("doubleLock", 1, &GetAndLockOptions{ + RetryStrategy: NewFailFastRetryStrategy(), + }) if err == nil { t.Fatalf("Expected GetAndLock to fail") } - if !IsTemporaryFailureError(err) { - t.Fatalf("Expected error to be TempFailError but was %v", err) + // The server and the mock do not agree on the error for locked documents. + if !IsKeyLockedError(err) && !IsTemporaryFailureError(err) { + t.Fatalf("Expected error to be TempFailError or IsKeyLockedError but was %s", reflect.TypeOf(err).String()) } } diff --git a/collection_subdoc.go b/collection_subdoc.go index 320fd856..b909d765 100644 --- a/collection_subdoc.go +++ b/collection_subdoc.go @@ -14,9 +14,10 @@ type LookupInSpec struct { // LookupInOptions are the set of options available to LookupIn. type LookupInOptions struct { - Context context.Context - Timeout time.Duration - Serializer JSONSerializer + Context context.Context + Timeout time.Duration + Serializer JSONSerializer + RetryStrategy RetryStrategy } // GetSpecOptions are the options available to LookupIn subdoc Get operations. @@ -152,12 +153,18 @@ func (c *Collection) lookupIn(ctx context.Context, id string, ops []LookupInSpec serializer = &DefaultJSONSerializer{} } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + ctrl := c.newOpManager(ctx) err = ctrl.wait(agent.LookupInEx(gocbcore.LookupInOptions{ Key: []byte(id), Ops: subdocs, CollectionName: c.name(), ScopeName: c.scopeName(), + RetryStrategy: retryWrapper, }, func(res *gocbcore.LookupInResult, err error) { if err != nil && !gocbcore.IsErrorStatus(err, gocbcore.StatusSubDocBadMulti) { errOut = maybeEnhanceKVErr(err, id, false) @@ -230,6 +237,7 @@ type MutateInOptions struct { DurabilityLevel DurabilityLevel StoreSemantic StoreSemantics Serializer JSONSerializer + RetryStrategy RetryStrategy // Internal: This should never be used and is not supported. AccessDeleted bool } @@ -728,6 +736,11 @@ func (c *Collection) mutate(ctx context.Context, id string, ops []MutateInSpec, }) } + retryWrapper := c.sb.RetryStrategyWrapper + if opts.RetryStrategy != nil { + retryWrapper = newRetryStrategyWrapper(opts.RetryStrategy) + } + coerced, durabilityTimeout := c.durabilityTimeout(ctx, opts.DurabilityLevel) if coerced { var cancel context.CancelFunc @@ -746,6 +759,7 @@ func (c *Collection) mutate(ctx context.Context, id string, ops []MutateInSpec, ScopeName: c.scopeName(), DurabilityLevel: gocbcore.DurabilityLevel(opts.DurabilityLevel), DurabilityLevelTimeout: durabilityTimeout, + RetryStrategy: retryWrapper, }, func(res *gocbcore.MutateInResult, err error) { if err != nil { errOut = maybeEnhanceKVErr(err, id, isInsertDocument) diff --git a/error.go b/error.go index 1f5aeef7..2efd8c59 100644 --- a/error.go +++ b/error.go @@ -126,22 +126,66 @@ func (err durabilityError) DurabilityError() bool { return true } +// TimeoutErrorWithDetail occurs when an operation times out. +// This error type contains extra details about why the operation +// timed out. +type TimeoutErrorWithDetail interface { + Timeout() bool + OperationID() string + RetryAttempts() uint32 + RetryReasons() []RetryReason +} + // TimeoutError occurs when an operation times out. type TimeoutError interface { Timeout() bool } type timeoutError struct { + operationID string + retryReasons []gocbcore.RetryReason + retryAttempts uint32 } func (err timeoutError) Error() string { - return "operation timed out" + base := "operation timed out" + if err.operationID != "" { + base = fmt.Sprintf("%s, lastOperationID: %s", base, err.operationID) + } + if err.retryAttempts > 0 { + base = fmt.Sprintf("%s, retried: %d", base, err.retryAttempts) + } + if len(err.retryReasons) > 0 { + var reasons []string + for _, reason := range err.retryReasons { + reasons = append(reasons, reason.Description()) + } + base = fmt.Sprintf("%s, retryReasons: [%s]", base, strings.Join(reasons, ",")) + } + + return base } func (err timeoutError) Timeout() bool { return true } +func (err timeoutError) OperationID() string { + return err.operationID +} + +func (err timeoutError) RetryAttempts() uint32 { + return err.retryAttempts +} + +func (err timeoutError) RetryReasons() []RetryReason { + var reasons []RetryReason + for _, reason := range err.retryReasons { + reasons = append(reasons, RetryReason(reason)) + } + return reasons +} + type serviceNotAvailableError struct { message string } @@ -993,14 +1037,6 @@ func (e analyticsQueryError) retryable() bool { return false } -// Timeout indicates whether or not this error is a timeout. -func (e analyticsQueryError) Timeout() bool { - if e.ErrorCode == 21002 { - return true - } - return false -} - // HTTPStatus returns the HTTP status code for the operation. func (e analyticsQueryError) HTTPStatus() int { return e.httpStatus @@ -1057,18 +1093,13 @@ func (e queryError) retryable() bool { return false } - if e.ErrorCode == 4050 || e.ErrorCode == 4070 || e.ErrorCode == 5000 { + if e.ErrorCode == 4040 || e.ErrorCode == 4050 || e.ErrorCode == 4070 { return true } - - return false -} - -// Timeout indicates whether or not this error is a timeout. -func (e queryError) Timeout() bool { - if e.ErrorCode == 1080 { + if e.ErrorCode == 5000 && strings.Contains(e.Message(), "queryport.indexNotFound") { return true } + return false } @@ -1153,7 +1184,7 @@ func (e searchMultiError) Errors() []SearchError { // PartialResults indicates whether or not the operation also yielded results. func (e searchMultiError) retryable() bool { - return e.httpStatus == 419 + return e.httpStatus == 429 } // ConfigurationError occurs when the client is configured incorrectly. diff --git a/go.mod b/go.mod index 1369c929..360bdb45 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/couchbase/gocb/v2 require ( - github.com/couchbase/gocbcore/v8 v8.0.0-beta.1 + github.com/couchbase/gocbcore/v8 v8.0.0-beta.1.0.20191028074748-28c6abe7087e github.com/couchbaselabs/gocbconnstr v1.0.3 github.com/couchbaselabs/gojcbmock v1.0.3 github.com/couchbaselabs/jsonx v1.0.0 diff --git a/go.sum b/go.sum index 9bce6ce5..26add15d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/couchbase/gocbcore/v8 v8.0.0-beta.1 h1:LjAYD4af7mFaiVtg5cJ18g2NvM5BF3zz4sJ8F/zahc4= -github.com/couchbase/gocbcore/v8 v8.0.0-beta.1/go.mod h1:ffhLs3F/rbsp3S5L4C8vOGixYWbDYivgokZ+vTZivOo= +github.com/couchbase/gocbcore/v8 v8.0.0-beta.1.0.20191028074748-28c6abe7087e h1:gMnFjoE/fWOvvQvGYfc97ZukrHtHN6MIGSr8MEvJhYE= +github.com/couchbase/gocbcore/v8 v8.0.0-beta.1.0.20191028074748-28c6abe7087e/go.mod h1:ffhLs3F/rbsp3S5L4C8vOGixYWbDYivgokZ+vTZivOo= github.com/couchbaselabs/gocbconnstr v1.0.3 h1:rkHC5N0ecbZ1NU7671ubApRdhSVc4rsulTEQ0W8O1uw= github.com/couchbaselabs/gocbconnstr v1.0.3/go.mod h1:Mg0VKc6azyPXhSq4b/xwsrW30ORe+H5L5hucCweYhj8= github.com/couchbaselabs/gojcbmock v1.0.3 h1:Ueenew8eiSM50pgHWV9+LThTB/aG29M2+LrxdjKasyU= diff --git a/mock_test.go b/mock_test.go index 05f049a5..bcf14489 100644 --- a/mock_test.go +++ b/mock_test.go @@ -38,6 +38,31 @@ type mockPendingOp struct { cancelSuccess bool } +func (mpo *mockPendingOp) RetryAttempts() uint32 { + return 0 +} + +func (mpo *mockPendingOp) Identifier() string { + return "" +} + +func (mpo *mockPendingOp) Idempotent() bool { + return false +} + +func (mpo *mockPendingOp) RetryReasons() []gocbcore.RetryReason { + return []gocbcore.RetryReason{} +} + +func (mpo *mockPendingOp) addRetryReason(reason gocbcore.RetryReason) { +} + +func (mpo *mockPendingOp) IncrementRetryAttempts() { +} + +func (mpo *mockPendingOp) SetCancelRetry(cancelFunc func() bool) { +} + func (mpo *mockPendingOp) Cancel() bool { return mpo.cancelSuccess } @@ -339,7 +364,7 @@ func (mko *mockKvProvider) GetOneReplicaEx(opts gocbcore.GetOneReplicaOptions, c return &mockPendingOp{cancelSuccess: mko.opCancellationSuccess}, nil } -func (mko *mockKvProvider) PingKvEx(opts gocbcore.PingKvOptions, cb gocbcore.PingKvExCallback) (gocbcore.PendingOp, error) { +func (mko *mockKvProvider) PingKvEx(opts gocbcore.PingKvOptions, cb gocbcore.PingKvExCallback) (gocbcore.CancellablePendingOp, error) { time.AfterFunc(mko.opWait, func() { if mko.err == nil { cb(mko.value.(*gocbcore.PingKvResult), nil) @@ -359,6 +384,12 @@ func (p *mockHTTPProvider) DoHttpRequest(req *gocbcore.HttpRequest) (*gocbcore.H return p.doFn(req) } +func (p *mockHTTPProvider) MaybeRetryRequest(req gocbcore.RetryRequest, reason gocbcore.RetryReason, + strategy gocbcore.RetryStrategy, retryFunc func()) bool { + time.AfterFunc(1*time.Millisecond, retryFunc) + return true +} + func (p *mockHTTPProvider) SupportsClusterCapability(capability gocbcore.ClusterCapability) bool { return p.supportFn(capability) } diff --git a/queryoptions.go b/queryoptions.go index 4930ce00..efdaa7e3 100644 --- a/queryoptions.go +++ b/queryoptions.go @@ -56,7 +56,8 @@ type QueryOptions struct { // JSONSerializer is used to deserialize each row in the result. This should be a JSON deserializer as results are JSON. // NOTE: if not set then query will always default to DefaultJSONSerializer. - Serializer JSONSerializer + Serializer JSONSerializer + RetryStrategy RetryStrategy } func (opts *QueryOptions) toMap(statement string) (map[string]interface{}, error) { diff --git a/retry.go b/retry.go new file mode 100644 index 00000000..d94bdabc --- /dev/null +++ b/retry.go @@ -0,0 +1,195 @@ +package gocb + +import ( + "time" + + "github.com/couchbase/gocbcore/v8" +) + +// RetryRequest is a request that can possibly be retried. +type RetryRequest interface { + RetryAttempts() uint32 + Identifier() string + Idempotent() bool + RetryReasons() []RetryReason +} + +type retryRequest struct { + attempts uint32 + identifier string + idempotent bool + reasons []RetryReason +} + +func (mgr *retryRequest) RetryAttempts() uint32 { + return mgr.attempts +} + +func (mgr *retryRequest) IncrementRetryAttempts() { + mgr.attempts++ +} + +func (mgr *retryRequest) Identifier() string { + return mgr.identifier +} + +func (mgr *retryRequest) Idempotent() bool { + return mgr.idempotent +} + +func (mgr *retryRequest) RetryReasons() []RetryReason { + return mgr.reasons +} + +// RetryReason represents the reason for an operation possibly being retried. +type RetryReason interface { + AllowsNonIdempotentRetry() bool + AlwaysRetry() bool + Description() string +} + +var ( + // UnknownRetryReason indicates that the operation failed for an unknown reason. + UnknownRetryReason = RetryReason(gocbcore.UnknownRetryReason) + + // SocketNotAvailableRetryReason indicates that the operation failed because the underlying socket was not available. + SocketNotAvailableRetryReason = RetryReason(gocbcore.SocketNotAvailableRetryReason) + + // ServiceNotAvailableRetryReason indicates that the operation failed because the requested service was not available. + ServiceNotAvailableRetryReason = RetryReason(gocbcore.ServiceNotAvailableRetryReason) + + // NodeNotAvailableRetryReason indicates that the operation failed because the requested node was not available. + NodeNotAvailableRetryReason = RetryReason(gocbcore.NodeNotAvailableRetryReason) + + // KVNotMyVBucketRetryReason indicates that the operation failed because it was sent to the wrong node for the vbucket. + KVNotMyVBucketRetryReason = RetryReason(gocbcore.KVNotMyVBucketRetryReason) + + // KVCollectionOutdatedRetryReason indicates that the operation failed because the collection ID on the request is outdated. + KVCollectionOutdatedRetryReason = RetryReason(gocbcore.KVCollectionOutdatedRetryReason) + + // KVErrMapRetryReason indicates that the operation failed for an unsupported reason but the KV error map indicated + // that the operation can be retried. + KVErrMapRetryReason = RetryReason(gocbcore.KVErrMapRetryReason) + + // KVLockedRetryReason indicates that the operation failed because the document was locked. + KVLockedRetryReason = RetryReason(gocbcore.KVLockedRetryReason) + + // KVTemporaryFailureRetryReason indicates that the operation failed because of a temporary failure. + KVTemporaryFailureRetryReason = RetryReason(gocbcore.KVTemporaryFailureRetryReason) + + // KVSyncWriteInProgressRetryReason indicates that the operation failed because a sync write is in progress. + KVSyncWriteInProgressRetryReason = RetryReason(gocbcore.KVSyncWriteInProgressRetryReason) + + // KVSyncWriteRecommitInProgressRetryReason indicates that the operation failed because a sync write recommit is in progress. + KVSyncWriteRecommitInProgressRetryReason = RetryReason(gocbcore.KVSyncWriteRecommitInProgressRetryReason) + + // ServiceResponseCodeIndicatedRetryReason indicates that the operation failed and the service responded stating that + // the request should be retried. + ServiceResponseCodeIndicatedRetryReason = RetryReason(gocbcore.ServiceResponseCodeIndicatedRetryReason) + + // SocketCloseInFlightRetryReason indicates that the operation failed because the socket was closed whilst the operation + // was in flight. + SocketCloseInFlightRetryReason = RetryReason(gocbcore.SocketCloseInFlightRetryReason) +) + +// RetryAction is used by a RetryStrategy to calculate the duration to wait before retrying an operation. +// Returning a value of 0 indicates to not retry. +type RetryAction interface { + Duration() time.Duration +} + +// NoRetryRetryAction represents an action that indicates to not retry. +type NoRetryRetryAction struct { +} + +// Duration is the length of time to wait before retrying an operation. +func (ra *NoRetryRetryAction) Duration() time.Duration { + return 0 +} + +// WithDurationRetryAction represents an action that indicates to retry with a given duration. +type WithDurationRetryAction struct { + WithDuration time.Duration +} + +// Duration is the length of time to wait before retrying an operation. +func (ra *WithDurationRetryAction) Duration() time.Duration { + return ra.WithDuration +} + +// RetryStrategy is to determine if an operation should be retried, and if so how long to wait before retrying. +type RetryStrategy interface { + RetryAfter(req RetryRequest, reason RetryReason) RetryAction +} + +func newRetryStrategyWrapper(strategy RetryStrategy) *retryStrategyWrapper { + return &retryStrategyWrapper{ + wrapped: strategy, + } +} + +type retryStrategyWrapper struct { + wrapped RetryStrategy +} + +// RetryAfter calculates and returns a RetryAction describing how long to wait before retrying an operation. +func (rs *retryStrategyWrapper) RetryAfter(req gocbcore.RetryRequest, reason gocbcore.RetryReason) gocbcore.RetryAction { + gocbRequest := &retryRequest{ + attempts: req.RetryAttempts(), + identifier: req.Identifier(), + idempotent: req.Idempotent(), + } + for _, retryReason := range req.RetryReasons() { + gocbReason, ok := retryReason.(RetryReason) + if !ok { + logErrorf("Failed to assert gocbcore retry reason to gocb retry reason: %v", reason) + continue + } + gocbRequest.reasons = append(gocbRequest.reasons, gocbReason) + } + + wrappedAction := rs.wrapped.RetryAfter(gocbRequest, RetryReason(reason)) + return wrappedAction +} + +// FailFastRetryStrategy represents a strategy that will never retry. +type FailFastRetryStrategy struct { +} + +// NewFailFastRetryStrategy returns a new FailFastRetryStrategy. +func NewFailFastRetryStrategy() *FailFastRetryStrategy { + return &FailFastRetryStrategy{} +} + +// RetryAfter calculates and returns a RetryAction describing how long to wait before retrying an operation. +func (rs *FailFastRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction { + return &NoRetryRetryAction{} +} + +// BackoffCalculator defines how backoff durations will be calculated by the retry API.g +type BackoffCalculator func(retryAttempts uint32) time.Duration + +// BestEffortRetryStrategy represents a strategy that will keep retrying until it succeeds (or the caller times out +// the request). +type BestEffortRetryStrategy struct { + backoffCalculator BackoffCalculator +} + +// NewBestEffortRetryStrategy returns a new BestEffortRetryStrategy which will use the supplied calculator function +// to calculate retry durations. If calculator is nil then a controlled backoff will be used. +func NewBestEffortRetryStrategy(calculator BackoffCalculator) *BestEffortRetryStrategy { + if calculator == nil { + calculator = gocbcore.ControlledBackoff + } + + return &BestEffortRetryStrategy{backoffCalculator: calculator} +} + +// RetryAfter calculates and returns a RetryAction describing how long to wait before retrying an operation. +func (rs *BestEffortRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction { + if req.Idempotent() || reason.AllowsNonIdempotentRetry() { + return &WithDurationRetryAction{WithDuration: rs.backoffCalculator(req.RetryAttempts())} + } + + return &NoRetryRetryAction{} +} diff --git a/retry_test.go b/retry_test.go new file mode 100644 index 00000000..256cace8 --- /dev/null +++ b/retry_test.go @@ -0,0 +1,158 @@ +package gocb + +import ( + "testing" + "time" + + "github.com/couchbase/gocbcore/v8" +) + +type mockGocbcoreRequest struct { + attempts uint32 + identifier string + idempotent bool + reasons []gocbcore.RetryReason + cancelSet bool + gocbcore.RetryRequest +} + +func (mgr *mockGocbcoreRequest) RetryAttempts() uint32 { + return mgr.attempts +} + +func (mgr *mockGocbcoreRequest) Identifier() string { + return mgr.identifier +} + +func (mgr *mockGocbcoreRequest) Idempotent() bool { + return mgr.idempotent +} + +func (mgr *mockGocbcoreRequest) RetryReasons() []gocbcore.RetryReason { + return mgr.reasons +} + +type mockRetryRequest struct { + attempts uint32 + identifier string + idempotent bool + reasons []RetryReason +} + +func (mgr *mockRetryRequest) RetryAttempts() uint32 { + return mgr.attempts +} + +func (mgr *mockRetryRequest) IncrementRetryAttempts() { + mgr.attempts++ +} + +func (mgr *mockRetryRequest) Identifier() string { + return mgr.identifier +} + +func (mgr *mockRetryRequest) Idempotent() bool { + return mgr.idempotent +} + +func (mgr *mockRetryRequest) RetryReasons() []RetryReason { + return mgr.reasons +} + +type mockRetryStrategy struct { + retried bool + action RetryAction +} + +func (mrs *mockRetryStrategy) RetryAfter(req RetryRequest, reason RetryReason) RetryAction { + mrs.retried = true + return mrs.action +} + +func mockBackoffCalculator(retryAttempts uint32) time.Duration { + return time.Millisecond * time.Duration(retryAttempts) +} + +func TestRetryWrapper_ForwardsAttempt(t *testing.T) { + expectedAction := &NoRetryRetryAction{} + strategy := newRetryStrategyWrapper(&mockRetryStrategy{action: expectedAction}) + + request := &mockGocbcoreRequest{ + reasons: []gocbcore.RetryReason{gocbcore.KVCollectionOutdatedRetryReason, gocbcore.UnknownRetryReason}, + } + action := strategy.RetryAfter(request, gocbcore.UnknownRetryReason) + if action != expectedAction { + t.Fatalf("Expected retry action to be %v but was %v", expectedAction, action) + } +} + +func TestBestEffortRetryStrategy_RetryAfterNoRetry(t *testing.T) { + strategy := NewBestEffortRetryStrategy(mockBackoffCalculator) + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.UnknownRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } +} + +func TestBestEffortRetryStrategy_RetryAfterAlwaysRetry(t *testing.T) { + strategy := NewBestEffortRetryStrategy(mockBackoffCalculator) + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.KVCollectionOutdatedRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } + + action = strategy.RetryAfter(&mockRetryRequest{attempts: 5}, RetryReason(gocbcore.KVCollectionOutdatedRetryReason)) + if action.Duration() != 5*time.Millisecond { + t.Fatalf("Expected duration to be %d but was %d", 5*time.Millisecond, action.Duration()) + } +} + +func TestBestEffortRetryStrategy_RetryAfterAllowsNonIdempotent(t *testing.T) { + strategy := NewBestEffortRetryStrategy(mockBackoffCalculator) + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.KVLockedRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } + + action = strategy.RetryAfter(&mockRetryRequest{attempts: 5}, RetryReason(gocbcore.KVLockedRetryReason)) + if action.Duration() != 5*time.Millisecond { + t.Fatalf("Expected duration to be %d but was %d", 5*time.Millisecond, action.Duration()) + } +} + +func TestBestEffortRetryStrategy_RetryAfterDefaultCalculator(t *testing.T) { + strategy := NewBestEffortRetryStrategy(nil) + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.KVCollectionOutdatedRetryReason)) + if action.Duration() != 1*time.Millisecond { + t.Fatalf("Expected duration to be %d but was %d", 1*time.Millisecond, action.Duration()) + } + + action = strategy.RetryAfter(&mockRetryRequest{attempts: 5}, RetryReason(gocbcore.KVLockedRetryReason)) + if action.Duration() != 1000*time.Millisecond { + t.Fatalf("Expected duration to be %d but was %d", 1000*time.Millisecond, action.Duration()) + } +} + +func TestFailFastRetryStrategy_RetryAfterNoRetry(t *testing.T) { + strategy := NewFailFastRetryStrategy() + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.UnknownRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } +} + +func TestFailFastRetryStrategy_RetryAfterAlwaysRetry(t *testing.T) { + strategy := NewFailFastRetryStrategy() + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.KVCollectionOutdatedRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } +} + +func TestFailFastRetryStrategy_RetryAfterAllowsNonIdempotent(t *testing.T) { + strategy := NewFailFastRetryStrategy() + action := strategy.RetryAfter(&mockRetryRequest{}, RetryReason(gocbcore.KVLockedRetryReason)) + if action.Duration() != 0 { + t.Fatalf("Expected duration to be %d but was %d", 0, action.Duration()) + } +} diff --git a/retrybehaviour.go b/retrybehaviour.go deleted file mode 100644 index 01e5f38f..00000000 --- a/retrybehaviour.go +++ /dev/null @@ -1,62 +0,0 @@ -package gocb - -import ( - "math" - "time" -) - -// retryBehavior defines the behavior to be used for retries -// Volatile: This API is subject to change at any time. -type retryBehavior interface { - NextInterval(retries uint) time.Duration - CanRetry(retries uint) bool -} - -// retryDelayFunction is called to get the next try delay -type retryDelayFunction func(retryDelay uint, retries uint) time.Duration - -// linearDelayFunction provides retry delay durations (ms) following a linear increment pattern -func linearDelayFunction(retryDelay uint, retries uint) time.Duration { - return time.Duration(retryDelay*retries) * time.Millisecond -} - -// exponentialDelayFunction provides retry delay durations (ms) following an exponential increment pattern -func exponentialDelayFunction(retryDelay uint, retries uint) time.Duration { - pow := math.Pow(float64(retryDelay), float64(retries)) - return time.Duration(pow) * time.Millisecond -} - -// delayRetryBehavior provides the behavior to use when retrying queries with a backoff delay -type delayRetryBehavior struct { - maxRetries uint - retryDelay uint - delayLimit time.Duration - delayFunc retryDelayFunction -} - -// standardDelayRetryBehavior provides a delayRetryBehavior that will retry at most maxRetries number of times and -// with an initial retry delay of retryDelay (ms) up to a maximum delay of delayLimit -func standardDelayRetryBehavior(maxRetries uint, retryDelay uint, delayLimit time.Duration, delayFunc retryDelayFunction) *delayRetryBehavior { - return &delayRetryBehavior{ - retryDelay: retryDelay, - maxRetries: maxRetries, - delayLimit: delayLimit, - delayFunc: delayFunc, - } -} - -// NextInterval calculates what the next retry interval (ms) should be given how many -// retries there have been already -func (rb *delayRetryBehavior) NextInterval(retries uint) time.Duration { - interval := rb.delayFunc(rb.retryDelay, retries) - if interval > rb.delayLimit { - interval = rb.delayLimit - } - - return interval -} - -// CanRetry determines whether or not the query can be retried according to the behavior -func (rb *delayRetryBehavior) CanRetry(retries uint) bool { - return retries < rb.maxRetries -} diff --git a/retrybehaviour_test.go b/retrybehaviour_test.go deleted file mode 100644 index eced2ed6..00000000 --- a/retrybehaviour_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package gocb - -import ( - "testing" - "time" -) - -func TestDelayRetryBehaviorCanRetry(t *testing.T) { - behav := standardDelayRetryBehavior(10, 1000, 1*time.Second, linearDelayFunction) - - var retries uint - if !behav.CanRetry(retries) { - t.Log("TestDelayRetryBehaviorLinear should have been able to retry but couldn't") - t.Fail() - } - - retries = 9 - if !behav.CanRetry(retries) { - t.Log("TestDelayRetryBehaviorLinear should have been able to retry but couldn't") - t.Fail() - } - - retries = 10 - if behav.CanRetry(retries) { - t.Log("TestDelayRetryBehaviorLinear shouldn't have been able to retry but could") - t.Fail() - } -} - -func TestDelayRetryBehaviorLinear(t *testing.T) { - behav := standardDelayRetryBehavior(10, 2, 500*time.Millisecond, linearDelayFunction) - - testNextInterval(t, behav, 1, 2*time.Millisecond, "TestDelayRetryBehaviorLinear") - testNextInterval(t, behav, 5, 10*time.Millisecond, "TestDelayRetryBehaviorLinear") - testNextInterval(t, behav, 10, 20*time.Millisecond, "TestDelayRetryBehaviorLinear") - testNextInterval(t, behav, 1000, 500*time.Millisecond, "TestDelayRetryBehaviorLinear") -} - -func TestDelayRetryBehaviorExponential(t *testing.T) { - behav := standardDelayRetryBehavior(10, 2, 500*time.Millisecond, exponentialDelayFunction) - - testNextInterval(t, behav, 1, 2*time.Millisecond, "TestDelayRetryBehaviorExponential") - testNextInterval(t, behav, 5, 32*time.Millisecond, "TestDelayRetryBehaviorExponential") - testNextInterval(t, behav, 10, 500*time.Millisecond, "TestDelayRetryBehaviorExponential") -} - -func testNextInterval(t *testing.T, behav *delayRetryBehavior, retries uint, expected time.Duration, testName string) { - interval := behav.NextInterval(retries) - if interval != expected { - t.Logf("%s expected interval of %v but was %v", testName, expected, interval) - t.Fail() - } -} diff --git a/searchquery_options.go b/searchquery_options.go index 0018a30e..71a60ec0 100644 --- a/searchquery_options.go +++ b/searchquery_options.go @@ -74,7 +74,8 @@ type SearchOptions struct { // JSONSerializer is used to deserialize each row in the result. This should be a JSON deserializer as results are JSON. // NOTE: if not set then query will always default to DefaultJSONSerializer. - Serializer JSONSerializer + Serializer JSONSerializer + RetryStrategy RetryStrategy } func (opts *SearchOptions) toOptionsData() (*searchQueryOptionsData, error) { diff --git a/stateblock.go b/stateblock.go index 74349c8a..b895ea31 100644 --- a/stateblock.go +++ b/stateblock.go @@ -28,10 +28,6 @@ type stateBlock struct { PersistTo uint ReplicateTo uint - N1qlRetryBehavior retryBehavior - AnalyticsRetryBehavior retryBehavior - SearchRetryBehavior retryBehavior - QueryTimeout time.Duration AnalyticsTimeout time.Duration SearchTimeout time.Duration @@ -42,6 +38,8 @@ type stateBlock struct { Transcoder Transcoder Serializer JSONSerializer + + RetryStrategyWrapper *retryStrategyWrapper } func (sb *stateBlock) getCachedClient() client { diff --git a/testdata/beer_sample_analytics_temp_error.json b/testdata/beer_sample_analytics_temp_error.json new file mode 100644 index 00000000..dbec8cbd --- /dev/null +++ b/testdata/beer_sample_analytics_temp_error.json @@ -0,0 +1,18 @@ +{ + "requestID": "fbe9ac66-a7ed-4b09-b1dc-4d3c791d8953", + "clientContextID": "62d29101-0c9f-400d-af2b-9bd44a557a7c", + "errors": [ + { + "code": 23000, + "msg": "Analytics Service is temporarily unavailable" + } + ], + "status": "errors", + "metrics": { + "elapsedTime": "837.425µs", + "executionTime": "732.345µs", + "resultCount": 0, + "resultSize": 0, + "errorCount": 1 + } +} diff --git a/viewquery_options.go b/viewquery_options.go index 7bb42444..1cf9ee3d 100644 --- a/viewquery_options.go +++ b/viewquery_options.go @@ -70,6 +70,8 @@ type ViewOptions struct { // JSONSerializer is used to deserialize each row in the result. This should be a JSON deserializer as results are JSON. // NOTE: if not set then views will always default to DefaultJSONSerializer. Serializer JSONSerializer + + RetryStrategy RetryStrategy } func (opts *ViewOptions) toURLValues() (*url.Values, error) {