Skip to content

Commit

Permalink
Introduce MaxDelay property to RetryStrategyOptions
Browse files Browse the repository at this point in the history
  • Loading branch information
@martintmk
martintmk committed Sep 22, 2023
1 parent a511f53 commit 222958e
Show file tree
Hide file tree
Showing 7 changed files with 210 additions and 52 deletions.
2 changes: 2 additions & 0 deletions src/Polly.Core/PublicAPI.Unshipped.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ Polly.Retry.RetryStrategyOptions<TResult>.Delay.get -> System.TimeSpan
Polly.Retry.RetryStrategyOptions<TResult>.Delay.set -> void
Polly.Retry.RetryStrategyOptions<TResult>.DelayGenerator.get -> System.Func<Polly.Retry.RetryDelayGeneratorArguments<TResult>, System.Threading.Tasks.ValueTask<System.TimeSpan?>>?
Polly.Retry.RetryStrategyOptions<TResult>.DelayGenerator.set -> void
Polly.Retry.RetryStrategyOptions<TResult>.MaxDelay.get -> System.TimeSpan?
Polly.Retry.RetryStrategyOptions<TResult>.MaxDelay.set -> void
Polly.Retry.RetryStrategyOptions<TResult>.MaxRetryAttempts.get -> int
Polly.Retry.RetryStrategyOptions<TResult>.MaxRetryAttempts.set -> void
Polly.Retry.RetryStrategyOptions<TResult>.OnRetry.get -> System.Func<Polly.Retry.OnRetryArguments<TResult>, System.Threading.Tasks.ValueTask>?
Expand Down
45 changes: 41 additions & 4 deletions src/Polly.Core/Retry/RetryHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,38 @@ internal static class RetryHelper
{
private const double JitterFactor = 0.5;

private const double MaxDelayJitterFactor = JitterFactor / 2.0;

private const double ExponentialFactor = 2.0;

public static bool IsValidDelay(TimeSpan delay) => delay >= TimeSpan.Zero;

public static TimeSpan GetRetryDelay(DelayBackoffType type, bool jitter, int attempt, TimeSpan baseDelay, ref double state, Func<double> randomizer)
public static TimeSpan GetRetryDelay(
DelayBackoffType type,
bool jitter,
int attempt,
TimeSpan baseDelay,
TimeSpan? maxDelay,
ref double state,
Func<double> randomizer)
{
try
{
return GetRetryDelayCore(type, jitter, attempt, baseDelay, ref state, randomizer);
var delay = GetRetryDelayCore(type, jitter, attempt, baseDelay, ref state, randomizer);

// stryker disable once equality : no means to test this
if (maxDelay is TimeSpan maxDelayValue && delay > maxDelayValue)
{
if (jitter)
{
// Apply jitter also to max delay, we are only allowed to substract from the delay not add to it.
return ApplyJitterForMaxDelay(maxDelayValue, randomizer);
}

return maxDelay.Value;
}

return delay;
}
catch (OverflowException)
{
Expand Down Expand Up @@ -89,7 +112,7 @@ private static TimeSpan DecorrelatedJitterBackoffV2(int attempt, TimeSpan baseDe
long targetTicksFirstDelay = baseDelay.Ticks;

double t = attempt + randomizer();
double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(PFactor * t));
double next = Math.Pow(ExponentialFactor, t) * Math.Tanh(Math.Sqrt(PFactor * t));

double formulaIntrinsicValue = next - prev;
prev = next;
Expand All @@ -98,5 +121,19 @@ private static TimeSpan DecorrelatedJitterBackoffV2(int attempt, TimeSpan baseDe
}

private static TimeSpan ApplyJitter(TimeSpan delay, Func<double> randomizer)
=> TimeSpan.FromMilliseconds(delay.TotalMilliseconds + ((delay.TotalMilliseconds * JitterFactor) * randomizer()));
{
var offset = (delay.TotalMilliseconds * JitterFactor) / 2;
var randomDelay = (delay.TotalMilliseconds * JitterFactor * randomizer()) - offset;
var newDelay = delay.TotalMilliseconds + randomDelay;

return TimeSpan.FromMilliseconds(newDelay);
}

private static TimeSpan ApplyJitterForMaxDelay(TimeSpan delay, Func<double> randomizer)
{
var randomDelay = delay.TotalMilliseconds * MaxDelayJitterFactor * randomizer();
var newDelay = delay.TotalMilliseconds - randomDelay;

return TimeSpan.FromMilliseconds(newDelay);
}
}
5 changes: 4 additions & 1 deletion src/Polly.Core/Retry/RetryResilienceStrategy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public RetryResilienceStrategy(
{
ShouldHandle = options.ShouldHandle;
BaseDelay = options.Delay;
MaxDelay = options.MaxDelay;
BackoffType = options.BackoffType;
RetryCount = options.MaxRetryAttempts;
OnRetry = options.OnRetry;
Expand All @@ -28,6 +29,8 @@ public RetryResilienceStrategy(

public TimeSpan BaseDelay { get; }

public TimeSpan? MaxDelay { get; }

public DelayBackoffType BackoffType { get; }

public int RetryCount { get; }
Expand Down Expand Up @@ -61,7 +64,7 @@ protected internal override async ValueTask<Outcome<T>> ExecuteCore<TState>(Func
return outcome;
}

var delay = RetryHelper.GetRetryDelay(BackoffType, UseJitter, attempt, BaseDelay, ref retryState, _randomizer);
var delay = RetryHelper.GetRetryDelay(BackoffType, UseJitter, attempt, BaseDelay, MaxDelay, ref retryState, _randomizer);
if (DelayGenerator is not null)
{
var delayArgs = new RetryDelayGeneratorArguments<T>(context, outcome, attempt);
Expand Down
18 changes: 16 additions & 2 deletions src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

namespace Polly.Retry;

#pragma warning disable IL2026 // Addressed with DynamicDependency on ValidationHelper.Validate method

/// <summary>
/// Represents the options used to configure a retry strategy.
/// </summary>
Expand Down Expand Up @@ -49,7 +51,6 @@ public class RetryStrategyOptions<TResult> : ResilienceStrategyOptions
/// </value>
public bool UseJitter { get; set; }

#pragma warning disable IL2026 // Addressed with DynamicDependency on ValidationHelper.Validate method
/// <summary>
/// Gets or sets the base delay between retries.
/// </summary>
Expand All @@ -73,7 +74,20 @@ public class RetryStrategyOptions<TResult> : ResilienceStrategyOptions
/// </value>
[Range(typeof(TimeSpan), "00:00:00", "1.00:00:00")]
public TimeSpan Delay { get; set; } = RetryConstants.DefaultBaseDelay;
#pragma warning restore IL2026

/// <summary>
/// Gets or sets the maximum delay between retries.
/// </summary>
/// <remarks>
/// This property is used to cap maximum delay between retries. It is useful when you want to limit the maximum delay after certain
/// number of between retries when it could reach a unreasonably high values, especially if <see cref="DelayBackoffType.Exponential"/> backoff is used.
/// If not specified, the delay is not capped. This property is ignored for delays generated by <see cref="DelayGenerator"/>.
/// </remarks>
/// <value>
/// The default value is <see langword="null"/>.
/// </value>
[Range(typeof(TimeSpan), "00:00:00", "1.00:00:00")]
public TimeSpan? MaxDelay { get; set; }

/// <summary>
/// Gets or sets a predicate that determines whether the retry should be executed for a given outcome.
Expand Down
164 changes: 120 additions & 44 deletions test/Polly.Core.Tests/Retry/RetryHelperTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,79 +28,155 @@ public void UnsupportedRetryBackoffType_Throws(bool jitter)
Assert.Throws<ArgumentOutOfRangeException>(() =>
{
double state = 0;
return RetryHelper.GetRetryDelay(type, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer);
return RetryHelper.GetRetryDelay(type, jitter, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer);
});
}

[InlineData(true)]
[InlineData(false)]
[Theory]
public void Constant_Ok(bool jitter)
[Fact]
public void Constant_Ok()
{
double state = 0;
if (jitter)
{
_randomizer = () => 0.5;
}

RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 0, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 1, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 2, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);

RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 1, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
}

var expected = !jitter ? TimeSpan.FromSeconds(1) : TimeSpan.FromSeconds(1.25);
[Fact]
public void Constant_Jitter_Ok()
{
double state = 0;

RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, true, 0, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, true, 1, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, true, 2, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);

_randomizer = () => 0.0;
RetryHelper
.GetRetryDelay(DelayBackoffType.Constant, true, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(0.75));

_randomizer = () => 0.4;
RetryHelper
.GetRetryDelay(DelayBackoffType.Constant, true, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(0.95));

_randomizer = () => 0.6;
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, true, 1, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(1.05));

_randomizer = () => 1.0;
RetryHelper
.GetRetryDelay(DelayBackoffType.Constant, true, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(1.25));
}

[InlineData(true)]
[InlineData(false)]
[Theory]
public void Linear_Ok(bool jitter)
[Fact]
public void Linear_Ok()
{
double state = 0;

RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 0, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 1, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 2, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);

if (jitter)
{
_randomizer = () => 0.5;
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 1, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, false, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3));
}

RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1.25));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2.5));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3.75));
}
else
{
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2));
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3));
}
[Fact]
public void Linear_Jitter_Ok()
{
double state = 0;

RetryHelper.GetRetryDelay(DelayBackoffType.Linear, true, 0, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, true, 1, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);

_randomizer = () => 0.0;
RetryHelper
.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(2.25));

_randomizer = () => 0.4;
RetryHelper
.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(2.85));

_randomizer = () => 0.5;
RetryHelper
.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(3));

_randomizer = () => 0.6;
RetryHelper.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(3.15));

_randomizer = () => 1.0;
RetryHelper
.GetRetryDelay(DelayBackoffType.Linear, true, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer)
.Should()
.Be(TimeSpan.FromSeconds(3.75));
}

[Fact]
public void Exponential_Ok()
{
double state = 0;

RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 0, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 2, TimeSpan.Zero, null, ref state, _randomizer).Should().Be(TimeSpan.Zero);

RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 0, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2));
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 2, TimeSpan.FromSeconds(1), null, ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(4));
}

[InlineData(DelayBackoffType.Linear, false)]
[InlineData(DelayBackoffType.Exponential, false)]
[InlineData(DelayBackoffType.Constant, false)]
[InlineData(DelayBackoffType.Linear, true)]
[InlineData(DelayBackoffType.Exponential, true)]
[InlineData(DelayBackoffType.Constant, true)]
[Theory]
public void MaxDelay_Ok(DelayBackoffType type, bool jitter)
{
_randomizer = () => 0.5;
var expected = jitter ? TimeSpan.FromSeconds(0.875) : TimeSpan.FromSeconds(1);
double state = 0;

RetryHelper.GetRetryDelay(type, jitter, 2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected);
}

[Fact]
public void MaxDelay_DelayLessThanMaxDelay_Respected()
{
double state = 0;
var expected = TimeSpan.FromSeconds(1);

RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1));
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2));
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(4));
RetryHelper.GetRetryDelay(DelayBackoffType.Constant, false, 2, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), ref state, _randomizer).Should().Be(expected);
}

[Fact]
public void GetRetryDelay_Overflow_ReturnsMaxTimeSpan()
{
double state = 0;

RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1000, TimeSpan.FromDays(1), ref state, _randomizer).Should().Be(TimeSpan.MaxValue);
RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, false, 1000, TimeSpan.FromDays(1), null, ref state, _randomizer).Should().Be(TimeSpan.MaxValue);
}

[InlineData(1)]
Expand Down Expand Up @@ -144,7 +220,7 @@ private static IReadOnlyList<TimeSpan> GetExponentialWithJitterBackoff(bool cont

for (int i = 0; i < retryCount; i++)
{
result.Add(RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, true, i, baseDelay, ref state, random));
result.Add(RetryHelper.GetRetryDelay(DelayBackoffType.Exponential, true, i, baseDelay, null, ref state, random));
}

return result;
Expand Down
24 changes: 24 additions & 0 deletions test/Polly.Core.Tests/Retry/RetryResilienceStrategyTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,30 @@ public async void OnRetry_EnsureCorrectArguments()
delays[2].Should().Be(TimeSpan.FromSeconds(6));
}

[Fact]
public async void MaxDelay_EnsureRespected()
{
var delays = new List<TimeSpan>();
_options.OnRetry = args =>
{
delays.Add(args.RetryDelay);
return default;
};

_options.ShouldHandle = args => PredicateResult.True();
_options.MaxRetryAttempts = 3;
_options.BackoffType = DelayBackoffType.Linear;
_options.MaxDelay = TimeSpan.FromMilliseconds(123);

var sut = CreateSut();

await ExecuteAndAdvance(sut);

delays[0].Should().Be(TimeSpan.FromMilliseconds(123));
delays[1].Should().Be(TimeSpan.FromMilliseconds(123));
delays[2].Should().Be(TimeSpan.FromMilliseconds(123));
}

[Fact]
public async Task OnRetry_EnsureExecutionTime()
{
Expand Down
Loading

0 comments on commit 222958e

Please sign in to comment.