Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 15 additions & 31 deletions apps/swift-ios/App/NativeFeatureClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1601,9 +1601,8 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
)
} catch {
// A connection can disappear after the server accepted the command
// but before its reply reaches us. Bootstrap expansion creates the
// thread before dispatching the stable final turn, so recover an
// interrupted empty thread by sending only that original turn.
// but before its reply reaches us. Confirm the original message
// before recovery. An empty worktree thread can still be in setup.
let recovered = try await recoverBootstrap(
client: client,
pending: pending,
Expand All @@ -1616,11 +1615,12 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
context: context
)
guard recovered else {
await resetFailedBootstrapIfConfirmed(
client: client,
pending: pending,
projectCwd: routedProject.workspaceRoot
)
if pending.worktreeBranchName == nil {
await resetFailedLocalBootstrapIfConfirmed(
client: client,
pending: pending
)
}
throw error
}
}
Expand Down Expand Up @@ -1686,6 +1686,9 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
}) {
return true
}
// Setup and its cancellation cleanup own the worktree until the first
// turn is committed. Sending a bare turn can race either operation.
guard pending.worktreeBranchName == nil else { return false }
guard snapshot.thread.projectId == projectID,
snapshot.thread.deletedAt == nil,
snapshot.thread.messages.isEmpty else {
Expand Down Expand Up @@ -1717,36 +1720,17 @@ final class NativeFeatureClient: FeatureClient, FeatureDeviceManaging,
return true
}

/// A failed bootstrap can leave its generated worktree behind after the
/// server rolls back the thread. Only reset the retry identity after a
/// fresh shell confirms the thread is absent; ambiguous network failures
/// keep the stable IDs so the normal recovery path remains idempotent.
private func resetFailedBootstrapIfConfirmed(
/// Reset a local retry only when a fresh shell confirms its thread is gone.
/// Worktree setup and cleanup stay server-owned and keep their stable IDs.
private func resetFailedLocalBootstrapIfConfirmed(
client: T3Client,
pending: PendingBootstrapSubmission,
projectCwd: String
pending: PendingBootstrapSubmission
) async {
guard let shell = try? await client.shellSnapshot(),
!shell.threads.contains(where: { $0.id == pending.threadID }) else {
return
}

if let branch = pending.worktreeBranchName,
let refs = try? await client.listVCSRefs(
cwd: projectCwd,
query: branch,
refresh: true,
limit: 100
),
let path = refs.refs.first(where: {
$0.name == branch && $0.isRemote != true
})?.worktreePath {
// Never force-remove: setup scripts may have left useful changes.
// A clean orphan is safe to reclaim; a dirty one remains visible
// through normal worktree management.
try? await client.removeWorktree(cwd: projectCwd, path: path)
}

removePendingBootstrap(identity: pending.identity)
}

Expand Down
9 changes: 7 additions & 2 deletions apps/swift-ios/Core/T3Client.swift
Original file line number Diff line number Diff line change
Expand Up @@ -773,14 +773,19 @@ public actor T3Client {
commandID: commandID,
messageID: messageID,
createdAt: createdAt
)
),
responseDeadline: worktreePreparation == nil ? .standard : .none
)
}

private func dispatchOverWebSocket(_ command: JSONValue) async throws -> DispatchResult {
private func dispatchOverWebSocket(
_ command: JSONValue,
responseDeadline: WebSocketRPCClient.ResponseDeadline = .standard
) async throws -> DispatchResult {
try await rpc.request(
RPCMethod.dispatchCommand.rawValue,
payload: command,
responseDeadline: responseDeadline,
as: DispatchResult.self
)
}
Expand Down
31 changes: 26 additions & 5 deletions apps/swift-ios/Core/WebSocketRPC.swift
Original file line number Diff line number Diff line change
Expand Up @@ -154,13 +154,20 @@ private struct RPCResponseEnvelope: Decodable, Sendable {
public actor WebSocketRPCClient {
public typealias EndpointProvider = @Sendable () async throws -> URL

public enum ResponseDeadline: Sendable, Equatable {
case standard
/// Setup can run until it completes, disconnects, or its caller cancels.
case none
}

private static let logger = Logger(
subsystem: "com.t3tools.t3code",
category: "WebSocketRPC"
)

private struct UnaryRequest {
let envelope: RPCRequestEnvelope
let responseDeadline: ResponseDeadline
var sent: Bool
var connectionWaitTask: Task<Void, Never>?
var sendDeadlineTask: Task<Void, Never>?
Expand Down Expand Up @@ -258,6 +265,7 @@ public actor WebSocketRPCClient {
private let endpointProvider: EndpointProvider
private let connectionWaitTimeout: Duration
private let responseTimeout: Duration
private let responseDeadlineSleep: @Sendable (Duration) async throws -> Void
private let keepaliveInterval: Duration
private let subscriptionBufferLimit: Int
private let reconnectBackoff: @Sendable (Int) -> Duration
Expand All @@ -283,6 +291,9 @@ public actor WebSocketRPCClient {
connector: any WebSocketConnecting = URLSessionWebSocketConnector(),
connectionWaitTimeout: Duration = .seconds(4),
responseTimeout: Duration = .seconds(30),
responseDeadlineSleep: @escaping @Sendable (Duration) async throws -> Void = {
try await Task.sleep(for: $0)
},
keepaliveInterval: Duration = .seconds(5),
subscriptionBufferLimit: Int = 1_024,
reconnectBackoff: @escaping @Sendable (Int) -> Duration = { failureCount in
Expand All @@ -298,6 +309,7 @@ public actor WebSocketRPCClient {
self.connector = connector
self.connectionWaitTimeout = connectionWaitTimeout
self.responseTimeout = responseTimeout
self.responseDeadlineSleep = responseDeadlineSleep
self.keepaliveInterval = keepaliveInterval > .zero ? keepaliveInterval : .seconds(5)
self.subscriptionBufferLimit = max(1, subscriptionBufferLimit)
self.reconnectBackoff = reconnectBackoff
Expand Down Expand Up @@ -397,17 +409,18 @@ public actor WebSocketRPCClient {
public func request<Result: Decodable & Sendable>(
_ tag: String,
payload: JSONValue = .object([:]),
responseDeadline: ResponseDeadline = .standard,
as type: Result.Type
) async throws -> Result {
let raw = try await requestRaw(tag, payload: payload)
let raw = try await requestRaw(tag, payload: payload, responseDeadline: responseDeadline)
return try raw.decode(type)
}

public func request(
_ tag: String,
payload: JSONValue = .object([:])
) async throws {
_ = try await requestRaw(tag, payload: payload)
_ = try await requestRaw(tag, payload: payload, responseDeadline: .standard)
}

public func subscribe<Value: Decodable & Sendable>(
Expand Down Expand Up @@ -524,7 +537,9 @@ public actor WebSocketRPCClient {
}
}

private func requestRaw(_ tag: String, payload: JSONValue) async throws -> JSONValue {
private func requestRaw(
_ tag: String, payload: JSONValue, responseDeadline: ResponseDeadline
) async throws -> JSONValue {
start()
let id = allocateRequestID()
let envelope = RPCRequestEnvelope(
Expand All @@ -543,6 +558,7 @@ public actor WebSocketRPCClient {
}
unary[id] = UnaryRequest(
envelope: envelope,
responseDeadline: responseDeadline,
sent: false,
connectionWaitTask: nil,
sendDeadlineTask: nil,
Expand Down Expand Up @@ -973,10 +989,15 @@ public actor WebSocketRPCClient {
request.connectionWaitTask = nil
request.sendDeadlineTask?.cancel()
request.sendDeadlineTask = nil
guard request.responseDeadline == .standard else {
unary[id] = request
return
}
let responseTimeout = responseTimeout
let sleep = responseDeadlineSleep
request.responseDeadlineTask = Task { [weak self] in
do {
try await Task.sleep(for: responseTimeout)
try await sleep(responseTimeout)
} catch {
return
}
Expand Down Expand Up @@ -1004,7 +1025,7 @@ public actor WebSocketRPCClient {
private func failUnaryOnSendDeadline(_ id: Int, connectionID: UUID) async {
guard let request = unary[id],
request.sent,
request.responseDeadlineTask == nil else { return }
request.sendDeadlineTask != nil else { return }
completeUnary(id, with: .failure(RPCError.responseTimedOut))
await disconnected(expectedConnectionID: connectionID)
}
Expand Down
99 changes: 97 additions & 2 deletions apps/swift-ios/Tests/CoreTests/WebSocketRPCRaceTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,68 @@ import XCTest

@MainActor
final class WebSocketRPCRaceTests: XCTestCase {
func testLongRunningResponseSurvivesTheNormalDeadline() async throws {
let connection = DeadlineWebSocketConnection(automaticallyReplies: false)
let clock = ManualResponseDeadlineClock()
let client = WebSocketRPCClient(
connector: SequencedConnector(connections: [connection]),
responseDeadlineSleep: { _ in await clock.wait() },
endpointProvider: { URL(string: "wss://studio.example/ws")! }
)
let setup = Task {
try await client.request("setup", responseDeadline: .none, as: JSONValue.self)
}
await connection.waitUntilRequestCount(1)
let ordinary = Task {
try await client.request("ordinary", as: JSONValue.self)
}
await connection.waitUntilRequestCount(2)
await clock.waitUntilEntered()
await clock.advance()
do {
_ = try await ordinary.value
XCTFail("Ordinary requests must keep their response deadline.")
} catch let error as RPCError {
guard case .responseTimedOut = error else { throw error }
}
await connection.waitUntilInterruptCount(1)
let interrupted = await connection.interruptedRequestIndices()
XCTAssertEqual(interrupted, [1], "The ordinary deadline must not interrupt setup.")
try await connection.replyToRequest(at: 0)
let result = try await setup.value
XCTAssertEqual(result, .object(["ok": .bool(true)]))
await client.stop()
}

func testLongRunningResponseStillEndsOnCancellationOrDisconnect() async throws {
for cancelled in [true, false] {
let connection = DeadlineWebSocketConnection(automaticallyReplies: false)
let client = WebSocketRPCClient(
connector: SequencedConnector(connections: [connection]),
endpointProvider: { URL(string: "wss://studio.example/ws")! }
)
let request = Task {
try await client.request("setup", responseDeadline: .none, as: JSONValue.self)
}
await connection.waitUntilRequestCount(1)
if cancelled { request.cancel() } else { await client.stop() }
do {
_ = try await request.value
XCTFail("Setup must stop when its owner cancels or disconnects.")
} catch is CancellationError {
XCTAssertTrue(cancelled)
} catch let error as RPCError {
guard case .disconnected = error, !cancelled else { throw error }
}
if cancelled {
await connection.waitUntilInterruptCount(1)
let interrupted = await connection.interruptedRequestIndices()
XCTAssertEqual(interrupted, [0])
}
await client.stop()
}
}

func testServerBatchesPreserveOrderAndSplitAtTheEventBudget() async throws {
let connection = SubscriptionTrafficConnection()
let client = WebSocketRPCClient(
Expand Down Expand Up @@ -365,7 +427,7 @@ final class WebSocketRPCRaceTests: XCTestCase {
)

let first = Task {
try await client.request("server.sendNeverReturns", as: JSONValue.self)
try await client.request("server.sendNeverReturns", responseDeadline: .none, as: JSONValue.self)
}
await hung.waitUntilSending()
do {
Expand Down Expand Up @@ -1327,14 +1389,47 @@ private actor BlockingStopConnection: WebSocketConnection {
}
}

private actor ManualResponseDeadlineClock {
private var entered = false
private var advanced = false
private var waiters: [CheckedContinuation<Void, Never>] = []
private var entryWaiters: [CheckedContinuation<Void, Never>] = []

func wait() async {
entered = true
entryWaiters.forEach { $0.resume() }
entryWaiters.removeAll()
guard !advanced else { return }
await withCheckedContinuation { waiters.append($0) }
}

func waitUntilEntered() async {
guard !entered else { return }
await withCheckedContinuation { entryWaiters.append($0) }
}

func advance() {
advanced = true
waiters.forEach { $0.resume() }
waiters.removeAll()
}
}

private actor DeadlineWebSocketConnection: WebSocketConnection {
private let automaticallyReplies: Bool
private var requestIDs: [Int] = []
private var interruptIDs: [Int] = []
private var requestWaiters: [(Int, CheckedContinuation<Void, Never>)] = []
private var interruptWaiters: [(Int, CheckedContinuation<Void, Never>)] = []
private var queuedResponses: [Data] = []
private var receiver: CheckedContinuation<Data, Error>?

init(automaticallyReplies: Bool = true) { self.automaticallyReplies = automaticallyReplies }

func interruptedRequestIndices() -> [Int] {
interruptIDs.compactMap { requestIDs.firstIndex(of: $0) }
}

func send(_ data: Data) throws {
let envelope = try JSONDecoder.t3.decode(JSONValue.self, from: data)
switch envelope["_tag"]?.stringValue {
Expand All @@ -1345,7 +1440,7 @@ private actor DeadlineWebSocketConnection: WebSocketConnection {
}
requestIDs.append(requestID)
resumeRequestWaiters()
if requestIDs.count > 1 {
if automaticallyReplies, requestIDs.count > 1 {
enqueue(try response(requestID: requestID))
}
case "Interrupt":
Expand Down
Loading
Loading