Add retries for RateLimitError errors and increase maximum retry in…

…terval to 30 seconds closes #208 (#209)

github/errors.go

@@ -15,10 +15,27 @@ func shouldRetryError(ctx context.Context, err error) bool {
 		if err.(*github.AbuseRateLimitError).RetryAfter != nil {
 			retryAfter = err.(*github.AbuseRateLimitError).RetryAfter
 		}
-		plugin.Logger(ctx).Debug("errors.shouldRetryError", "abuse_rate_limit_error", err, "retryAfter", retryAfter)
+		plugin.Logger(ctx).Debug("github_errors.shouldRetryError", "abuse_rate_limit_error", err, "retry_after", retryAfter)
 		return true
 	}
 
+	if _, ok := err.(*github.RateLimitError); ok {
+		// Get the limit reset timestamp if returned
+		var resetAfter time.Time
+		if err.(*github.RateLimitError).Rate.String() != "" {
+			resetAfter = err.(*github.RateLimitError).Rate.Reset.Time
+		}
+
+		// Get the remaining time
+		t1 := time.Now()
+		diff := resetAfter.Sub(t1).Seconds()
+		plugin.Logger(ctx).Debug("github_errors.shouldRetryError", "rate_limit_error", err, "reset_after", diff)
+
+		// Treat the error as non-fatal if the remaining time for limit reset is
+		// less than 60s
+		return diff <= 60
+	}
+
 	return false
 }
 

github/utils.go

@@ -137,15 +137,15 @@ func retryHydrate(ctx context.Context, d *plugin.QueryData, hydrateData *plugin.
 	interval := time.Duration(1)
 
 	// Create the backoff based on the given mode
-	// Use exponential instead of fibnacci due to GitHub's aggressive throttling
+	// Use exponential instead of fibonacci due to GitHub's aggressive throttling
 	backoff, err := retry.NewExponential(interval * time.Second)
 	if err != nil {
 		return nil, err
 	}
 
-	// Ensure the maximum value is 10s. In this scenario, the sleep values would be
-	// 1s, 2s, 4s, 8s, 10s, 10s...
-	backoff = retry.WithCappedDuration(10*time.Second, backoff)
+	// Ensure the maximum value is 30s. In this scenario, the sleep values would be
+	// 1s, 2s, 4s, 16s, 30s, 30s...
+	backoff = retry.WithCappedDuration(30*time.Second, backoff)
 
 	var hydrateResult interface{}