Skip to content

Computed retry delay #664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Apr 8, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nanpa/improve-reconnect-delay-logic.kdl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
patch type="changed" "Improve reconnect delay logic"
15 changes: 12 additions & 3 deletions Sources/LiveKit/Core/Room+Engine.swift
Original file line number Diff line number Diff line change
Expand Up @@ -358,8 +358,15 @@ extension Room {

do {
try await Task.retrying(totalAttempts: _state.connectOptions.reconnectAttempts,
retryDelay: _state.connectOptions.reconnectAttemptDelay)
{ currentAttempt, totalAttempts in
retryDelay: { @Sendable attempt in
let delay = TimeInterval.computeReconnectDelay(forAttempt: attempt,
baseDelay: self._state.connectOptions.reconnectAttemptDelay,
maxDelay: self._state.connectOptions.reconnectMaxDelay,
totalAttempts: self._state.connectOptions.reconnectAttempts,
addJitter: true)
self.log("[Connect] Retry cycle waiting for \(String(format: "%.2f", delay)) seconds before attempt \(attempt + 1)")
return delay
}) { currentAttempt, totalAttempts in

// Not reconnecting state anymore
guard let currentMode = self._state.isReconnectingWithMode else {
Expand All @@ -370,7 +377,7 @@ extension Room {
// Full reconnect failed, give up
guard currentMode != .full else { return }

self.log("[Connect] Retry in \(self._state.connectOptions.reconnectAttemptDelay) seconds, \(currentAttempt)/\(totalAttempts) tries left.")
self.log("[Connect] Starting retry attempt \(currentAttempt)/\(totalAttempts) with mode: \(currentMode)")

// Try full reconnect for the final attempt
if totalAttempts == currentAttempt, self._state.nextReconnectMode == nil {
Expand All @@ -387,8 +394,10 @@ extension Room {
do {
if case .quick = mode {
try await quickReconnectSequence()
self.log("[Connect] Quick reconnect succeeded for attempt \(currentAttempt)")
} else if case .full = mode {
try await fullReconnectSequence()
self.log("[Connect] Full reconnect succeeded for attempt \(currentAttempt)")
}
} catch {
self.log("[Connect] Reconnect mode: \(mode) failed with error: \(error)", .error)
Expand Down
60 changes: 59 additions & 1 deletion Sources/LiveKit/Extensions/TimeInterval.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,12 @@ import Foundation

/// Default timeout `TimeInterval`s used throughout the SDK.
public extension TimeInterval {
static let defaultReconnectAttemptDelay: Self = 2
// reconnection settings
static let defaultReconnectDelay: Self = 0.3 // 300ms to match JS SDK
// reconnect delays for the first few attempts, followed by maxRetryDelay
static let defaultReconnectMaxDelay: Self = 7 // maximum retry delay in seconds
static let defaultReconnectDelayJitter: Self = 1.0 // 1 second jitter for later retries

// the following 3 timeouts are used for a typical connect sequence
static let defaultSocketConnect: Self = 10
// used for validation mode
Expand All @@ -30,6 +35,59 @@ public extension TimeInterval {
static let resolveSid: Self = 7 + 5 // Join response + 5
static let defaultPublish: Self = 10
static let defaultCaptureStart: Self = 10

/// Computes a retry delay based on an "easeOutCirc" curve between baseDelay and maxDelay.
///
/// The easeOutCirc curve provides a dramatic early acceleration followed by a gentler approach to the maximum,
/// resulting in larger delays early in the reconnection sequence to reduce unnecessary network traffic.
///
/// Example values for 10 reconnection attempts with baseDelay=0.3s and maxDelay=7s:
/// - Attempt 0: ~0.85s (already 12% of the way to max)
/// - Attempt 1: ~2.2s (30% of the way to max)
/// - Attempt 2: ~3.4s (45% of the way to max)
/// - Attempt 5: ~5.9s (82% of the way to max)
/// - Attempt 9: 7.0s (exactly maxDelay)
///
/// - Parameter attempt: The current retry attempt (0-based index)
/// - Parameter baseDelay: The minimum delay for the curve's starting point (default: 0.3s)
/// - Parameter maxDelay: The maximum delay for the last retry attempt (default: 7s)
/// - Parameter totalAttempts: The total number of attempts that will be made (default: 10)
/// - Parameter addJitter: Whether to add random jitter to the delay (default: true)
/// - Returns: The delay in seconds to wait before the next retry attempt
@Sendable
static func computeReconnectDelay(forAttempt attempt: Int,
baseDelay: TimeInterval,
maxDelay: TimeInterval,
totalAttempts: Int,
addJitter: Bool = true) -> TimeInterval
{
// Last attempt should use maxDelay exactly
if attempt >= totalAttempts - 1 {
return maxDelay
}

// Make sure we have a valid value for total attempts
let validTotalAttempts = max(2, totalAttempts) // Need at least 2 attempts

// Apply easeOutCirc curve to all attempts (0 through n-2)
// We normalize the attempt index to a 0-1 range
let normalizedIndex = Double(attempt) / Double(validTotalAttempts - 1)

// Apply easeOutCirc curve: sqrt(1 - pow(x - 1, 2))
// This creates a very dramatic early acceleration with a smooth approach to the maximum
let t = normalizedIndex - 1.0
let easeOutCircProgress = sqrt(1.0 - t * t)

// Calculate the delay by applying the easeOutCirc curve between baseDelay and maxDelay
let calculatedDelay = baseDelay + easeOutCircProgress * (maxDelay - baseDelay)

// Add jitter if requested (up to 10% of the calculated delay)
if addJitter {
return calculatedDelay + (Double.random(in: 0 ..< 0.1) * calculatedDelay)
} else {
return calculatedDelay
}
}
}

extension TimeInterval {
Expand Down
18 changes: 15 additions & 3 deletions Sources/LiveKit/Support/AsyncRetry.swift
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,28 @@ extension Task where Failure == Error {
totalAttempts: Int = 3,
retryDelay: TimeInterval = 1,
@_implicitSelfCapture operation: @Sendable @escaping (_ currentAttempt: Int, _ totalAttempts: Int) async throws -> Success
) -> Task {
retrying(priority: priority, totalAttempts: totalAttempts,
retryDelay: { @Sendable _ in retryDelay },
operation: operation)
}

static func retrying(
priority: TaskPriority? = nil,
totalAttempts: Int,
retryDelay: @Sendable @escaping (_ attempt: Int) -> TimeInterval,
@_implicitSelfCapture operation: @Sendable @escaping (_ currentAttempt: Int, _ totalAttempts: Int) async throws -> Success
) -> Task {
Task(priority: priority) {
for currentAttempt in 1 ..< max(1, totalAttempts) {
print("[Retry] Attempt \(currentAttempt) of \(totalAttempts), delay: \(retryDelay)")
let delay = retryDelay(currentAttempt - 1)
print("[Retry] Attempt \(currentAttempt) of \(totalAttempts), delay: \(delay)")
do {
return try await operation(currentAttempt, totalAttempts)
} catch {
let oneSecond = TimeInterval(1_000_000_000)
let delayNS = UInt64(oneSecond * retryDelay)
print("[Retry] Waiting for \(retryDelay) seconds...")
let delayNS = UInt64(oneSecond * delay)
print("[Retry] Waiting for \(delay) seconds...")
try await Task<Never, Never>.sleep(nanoseconds: delayNS)
continue
}
Expand Down
2 changes: 2 additions & 0 deletions Sources/LiveKit/Types/Options/ConnectOptions+Copy.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ public extension ConnectOptions {
func copyWith(autoSubscribe: ValueOrAbsent<Bool> = .absent,
reconnectAttempts: ValueOrAbsent<Int> = .absent,
reconnectAttemptDelay: ValueOrAbsent<TimeInterval> = .absent,
reconnectMaxDelay: ValueOrAbsent<TimeInterval> = .absent,
protocolVersion: ValueOrAbsent<ProtocolVersion> = .absent) -> ConnectOptions
{
ConnectOptions(autoSubscribe: autoSubscribe.value(ifAbsent: self.autoSubscribe),
reconnectAttempts: reconnectAttempts.value(ifAbsent: self.reconnectAttempts),
reconnectAttemptDelay: reconnectAttemptDelay.value(ifAbsent: self.reconnectAttemptDelay),
reconnectMaxDelay: reconnectMaxDelay.value(ifAbsent: self.reconnectMaxDelay),
protocolVersion: protocolVersion.value(ifAbsent: self.protocolVersion))
}
}
39 changes: 34 additions & 5 deletions Sources/LiveKit/Types/Options/ConnectOptions.swift
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,34 @@ public final class ConnectOptions: NSObject, Sendable {
@objc
public let reconnectAttempts: Int

/// The delay between reconnect attempts.
/// The minimum delay value for reconnection attempts.
/// Default is 0.3 seconds (TimeInterval.defaultReconnectDelay).
///
/// This value serves as the starting point for the easeOutCirc reconnection curve.
/// See `reconnectMaxDelay` for more details on how the reconnection delay is calculated.
@objc
public let reconnectAttemptDelay: TimeInterval

/// The maximum delay between reconnect attempts.
/// Default is 7 seconds (TimeInterval.defaultReconnectMaxDelay).
///
/// The reconnection delay uses an "easeOutCirc" curve between reconnectAttemptDelay and reconnectMaxDelay:
/// - For all attempts except the last, the delay follows this curve
/// - The curve grows rapidly at first and then more gradually approaches the maximum
/// - The last attempt always uses exactly reconnectMaxDelay
///
/// Example for 10 reconnection attempts with baseDelay=0.3s and maxDelay=7s:
/// - Attempt 0: ~0.85s (already 12% of the way to max)
/// - Attempt 1: ~2.2s (30% of the way to max)
/// - Attempt 2: ~3.4s (45% of the way to max)
/// - Attempt 5: ~5.9s (82% of the way to max)
/// - Attempt 9: 7.0s (exactly maxDelay)
///
/// This approach provides larger delays early in the reconnection sequence to reduce
/// unnecessary network traffic when connections are likely to fail.
@objc
public let reconnectMaxDelay: TimeInterval

/// The timeout interval for the initial websocket connection.
@objc
public let socketConnectTimeoutInterval: TimeInterval
Expand All @@ -56,8 +80,9 @@ public final class ConnectOptions: NSObject, Sendable {
@objc
override public init() {
autoSubscribe = true
reconnectAttempts = 3
reconnectAttemptDelay = .defaultReconnectAttemptDelay
reconnectAttempts = 10
reconnectAttemptDelay = .defaultReconnectDelay
reconnectMaxDelay = .defaultReconnectMaxDelay
socketConnectTimeoutInterval = .defaultSocketConnect
primaryTransportConnectTimeout = .defaultTransportState
publisherTransportConnectTimeout = .defaultTransportState
Expand All @@ -68,8 +93,9 @@ public final class ConnectOptions: NSObject, Sendable {

@objc
public init(autoSubscribe: Bool = true,
reconnectAttempts: Int = 3,
reconnectAttemptDelay: TimeInterval = .defaultReconnectAttemptDelay,
reconnectAttempts: Int = 10,
reconnectAttemptDelay: TimeInterval = .defaultReconnectDelay,
reconnectMaxDelay: TimeInterval = .defaultReconnectMaxDelay,
socketConnectTimeoutInterval: TimeInterval = .defaultSocketConnect,
primaryTransportConnectTimeout: TimeInterval = .defaultTransportState,
publisherTransportConnectTimeout: TimeInterval = .defaultTransportState,
Expand All @@ -80,6 +106,7 @@ public final class ConnectOptions: NSObject, Sendable {
self.autoSubscribe = autoSubscribe
self.reconnectAttempts = reconnectAttempts
self.reconnectAttemptDelay = reconnectAttemptDelay
self.reconnectMaxDelay = max(reconnectMaxDelay, reconnectAttemptDelay)
self.socketConnectTimeoutInterval = socketConnectTimeoutInterval
self.primaryTransportConnectTimeout = primaryTransportConnectTimeout
self.publisherTransportConnectTimeout = publisherTransportConnectTimeout
Expand All @@ -95,6 +122,7 @@ public final class ConnectOptions: NSObject, Sendable {
return autoSubscribe == other.autoSubscribe &&
reconnectAttempts == other.reconnectAttempts &&
reconnectAttemptDelay == other.reconnectAttemptDelay &&
reconnectMaxDelay == other.reconnectMaxDelay &&
socketConnectTimeoutInterval == other.socketConnectTimeoutInterval &&
primaryTransportConnectTimeout == other.primaryTransportConnectTimeout &&
publisherTransportConnectTimeout == other.publisherTransportConnectTimeout &&
Expand All @@ -108,6 +136,7 @@ public final class ConnectOptions: NSObject, Sendable {
hasher.combine(autoSubscribe)
hasher.combine(reconnectAttempts)
hasher.combine(reconnectAttemptDelay)
hasher.combine(reconnectMaxDelay)
hasher.combine(socketConnectTimeoutInterval)
hasher.combine(primaryTransportConnectTimeout)
hasher.combine(publisherTransportConnectTimeout)
Expand Down
Loading
Loading