Strict concurrency for SelectableEventLoop & MTELG (apple#3084)

Motivation

SelectableEventLoop and MTEL are our major entry points for execution.
These need to be strict concurrency clean. Unfortunately, changes in one
tend to ripple into the other, so we need to tackle both at once.

Modifications

- A few closures get annotated @sendable
- Add a NIOLockedValueBox to convince the compiler that this use is
safe.
- Make a few types explicitly not Sendable

Result

No concurrency warnings in MTELG and SelectableEventLoop.

Author: Lukasa

Sources/NIOPosix/MultiThreadedEventLoopGroup.swift

@@ -40,6 +40,9 @@ struct NIORegistration: Registration {
     var registrationID: SelectorRegistrationID
 }
 
+@available(*, unavailable)
+extension NIORegistration: Sendable {}
+
 private let nextEventLoopGroupID = ManagedAtomic(0)
 
 /// Called per `NIOThread` that is created for an EventLoop to do custom initialization of the `NIOThread` before the actual `EventLoop` is run on it.
@@ -391,7 +394,7 @@ public final class MultiThreadedEventLoopGroup: EventLoopGroup {
             return
         }
 
-        var result: Result<Void, Error> = .success(())
+        let result: NIOLockedValueBox<Result<Void, Error>> = NIOLockedValueBox(.success(()))
 
         for loop in self.eventLoops {
             g.enter()
@@ -400,7 +403,9 @@ public final class MultiThreadedEventLoopGroup: EventLoopGroup {
                 case .success:
                     ()
                 case .failure(let error):
-                    result = .failure(error)
+                    result.withLockedValue {
+                        $0 = .failure(error)
+                    }
                 }
                 g.leave()
             }
@@ -418,14 +423,14 @@ public final class MultiThreadedEventLoopGroup: EventLoopGroup {
                             "MultiThreadedEventLoopGroup in illegal state when closing: \(self.runState)"
                         )
                     case .closing(let callbacks):
-                        let overallError: Error? = {
-                            switch result {
+                        let overallError: Error? = result.withLockedValue {
+                            switch $0 {
                             case .success:
                                 return nil
                             case .failure(let error):
                                 return error
                             }
-                        }()
+                        }
                         self.runState = .closed(overallError)
                         return (overallError, callbacks)
                     }
@@ -490,9 +495,9 @@ extension MultiThreadedEventLoopGroup: CustomStringConvertible {
 }
 
 @usableFromInline
-struct ErasedUnownedJob {
+struct ErasedUnownedJob: Sendable {
     @usableFromInline
-    let erasedJob: Any
+    let erasedJob: any Sendable
 
     @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *)
     init(job: UnownedJob) {
@@ -567,6 +572,12 @@ extension ScheduledTask: Comparable {
     }
 }
 
+@available(*, unavailable)
+extension ScheduledTask: Sendable {}
+
+@available(*, unavailable)
+extension ScheduledTask.Kind: Sendable {}
+
 extension NIODeadline {
     @inlinable
     func readyIn(_ target: NIODeadline) -> TimeAmount {

Sources/NIOPosix/SelectableEventLoop.swift

@@ -869,7 +869,10 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
         self._succeededVoidFuture = nil
     }
 
-    internal func initiateClose(queue: DispatchQueue, completionHandler: @escaping (Result<Void, Error>) -> Void) {
+    internal func initiateClose(
+        queue: DispatchQueue,
+        completionHandler: @escaping @Sendable (Result<Void, Error>) -> Void
+    ) {
         func doClose() {
             self.assertInEventLoop()
             self._parentGroup = nil  // break the cycle
@@ -948,7 +951,7 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
     }
 
     @usableFromInline
-    func shutdownGracefully(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) {
+    func shutdownGracefully(queue: DispatchQueue, _ callback: @escaping @Sendable (Error?) -> Void) {
         if self.canBeShutdownIndividually {
             self.initiateClose(queue: queue) { result in
                 self.syncFinaliseClose(joinThread: false)  // This thread was taken over by somebody else
@@ -1012,12 +1015,18 @@ enum UnderlyingTask {
     case callback(any NIOScheduledCallbackHandler)
 }
 
+@available(*, unavailable)
+extension UnderlyingTask: Sendable {}
+
 @usableFromInline
 internal enum LoopTask {
     case scheduled(ScheduledTask)
     case immediate(UnderlyingTask)
 }
 
+@available(*, unavailable)
+extension LoopTask: Sendable {}
+
 @inlinable
 internal func assertExpression(_ body: () -> Bool) {
     assert(