forked from openshift/aws-ebs-csi-driver
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add retry manager to reduce RateLimitExceeded errors
- Loading branch information
1 parent
1b242f8
commit 9514fab
Showing
3 changed files
with
102 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package cloud | ||
|
|
||
| import ( | ||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/aws/retry" | ||
| ) | ||
|
|
||
| const ( | ||
| // retryMaxAttempt sets max number of EC2 API call attempts. | ||
| // Set high enough to ensure default sidecar timeout will cancel context long before we stop retrying. | ||
| retryMaxAttempt = 50 | ||
| ) | ||
|
|
||
| // retryManager dictates the retry strategies of EC2 API calls. | ||
| // Each mutating EC2 API has its own retryer because the AWS SDK throttles on a retryer object level, not by API name. | ||
| // While default AWS accounts share request tokens between mutating APIs, users can raise limits for individual APIs. | ||
| // Separate retryers ensures that throttling one API doesn't unintentionally throttle others with separate token buckets. | ||
| type retryManager struct { | ||
| createVolumeRetryer aws.Retryer | ||
| deleteVolumeRetryer aws.Retryer | ||
| attachVolumeRetryer aws.Retryer | ||
| detachVolumeRetryer aws.Retryer | ||
| modifyVolumeRetryer aws.Retryer | ||
| createSnapshotRetryer aws.Retryer | ||
| deleteSnapshotRetryer aws.Retryer | ||
| enableFastSnapshotRestoresRetryer aws.Retryer | ||
| unbatchableDescribeVolumesModificationsRetryer aws.Retryer | ||
| } | ||
|
|
||
| func newRetryManager() *retryManager { | ||
| return &retryManager{ | ||
| createVolumeRetryer: newAdaptiveRetryer(), | ||
| attachVolumeRetryer: newAdaptiveRetryer(), | ||
| deleteVolumeRetryer: newAdaptiveRetryer(), | ||
| detachVolumeRetryer: newAdaptiveRetryer(), | ||
| modifyVolumeRetryer: newAdaptiveRetryer(), | ||
| createSnapshotRetryer: newAdaptiveRetryer(), | ||
| deleteSnapshotRetryer: newAdaptiveRetryer(), | ||
| enableFastSnapshotRestoresRetryer: newAdaptiveRetryer(), | ||
| unbatchableDescribeVolumesModificationsRetryer: newAdaptiveRetryer(), | ||
| } | ||
| } | ||
|
|
||
| // newAdaptiveRetryer restricts attempts of API calls that recently hit throttle errors. | ||
| func newAdaptiveRetryer() *retry.AdaptiveMode { | ||
| return retry.NewAdaptiveMode(func(ao *retry.AdaptiveModeOptions) { | ||
| ao.StandardOptions = append(ao.StandardOptions, func(so *retry.StandardOptions) { | ||
| so.MaxAttempts = retryMaxAttempt | ||
| }) | ||
| }) | ||
| } |