Remove temp_await and safe_async throughout the entire codebase [part 1] (swiftlang#7761)

Remove both `temp_await` and `safe_async` in favor of broader adoption of
`async`/`await`.

### Motivation:

`temp_await` is very dangerous API that blocks the current thread and is
not marked as unavailable from async methods. The only need for
temp_await is because LLBuild only supports non-async methods so any
llbuild commands that call async methods have to block waiting.

Because `temp_await` is liberally spread across the code base there are
a number of async methods that may indirectly call into `temp_await` and
indefinitely block a swift concurrency thread. This risks a deadlock. To
prevent the deadlock the `safe_async` method dispatches to a GCD
concurrent queue before executing the callback code. Because GCD will
create new threads if all existing threads are blocked (unlike swift
concurrency) this effectively prevents deadlock risks at the risk of
thread explosion.

It would be better if the codebase was free of both the risks of
deadlock and thread explosion. Additionally it would be nice if the APIs
using `temp_await` were marked async so that more parts of the code base
could safely adopt `async`/`await.`

### Modifications:

* To augment existing memoizer types new async types are added
`temp_dir` is replaced by `unsafe_await` which is limited to one used
which is called by llbuild
* `safe_async` is replaced by CheckedThrowingContinuation
* Multiple protocols have been modified to change their requirements from
sync to async
* Usage of DispatchSemaphore and DispatchGroup have been replaced with
structured concurrency
* Many methods that were non-async but used `temp_await` have been properly
marked as `async`

### Result:

No usage of `temp_dir` or `safe_async` and only one (safe) usage of`unsafe_await` from an LLBuild command

Author: AndrewHoos

Sources/Basics/Archiver/Archiver.swift

@@ -36,13 +36,10 @@ public protocol Archiver: Sendable {
     /// - Parameters:
     ///   - directory: The `AbsolutePath` to the archive to extract.
     ///   - destinationPath: The `AbsolutePath` to the directory to extract to.
-    ///   - completion: The completion handler that will be called when the operation finishes to notify of its success.
-    @available(*, noasync, message: "Use the async alternative")
     func compress(
         directory: AbsolutePath,
-        to destinationPath: AbsolutePath,
-        completion: @escaping @Sendable (Result<Void, Error>) -> Void
-    )
+        to destinationPath: AbsolutePath
+    ) async throws
 
     /// Asynchronously validates if a file is an archive.
     ///
@@ -71,20 +68,6 @@ extension Archiver {
         }
     }
 
-    /// Asynchronously compresses the contents of a directory to a destination archive.
-    ///
-    /// - Parameters:
-    ///   - directory: The `AbsolutePath` to the archive to extract.
-    ///   - destinationPath: The `AbsolutePath` to the directory to extract to.
-    public func compress(
-        directory: AbsolutePath,
-        to destinationPath: AbsolutePath
-    ) async throws {
-        try await withCheckedThrowingContinuation { continuation in
-            self.compress(directory: directory, to: destinationPath, completion: { continuation.resume(with: $0) })
-        }
-    }
-
     /// Asynchronously validates if a file is an archive.
     ///
     /// - Parameters:

Sources/Basics/Archiver/TarArchiver.swift

@@ -82,36 +82,29 @@ public struct TarArchiver: Archiver {
 
     public func compress(
         directory: AbsolutePath,
-        to destinationPath: AbsolutePath,
-        completion: @escaping @Sendable (Result<Void, Error>) -> Void
-    ) {
-        do {
-            guard self.fileSystem.isDirectory(directory) else {
-                throw FileSystemError(.notDirectory, directory.underlying)
-            }
+        to destinationPath: AbsolutePath
+    ) async throws {
 
-            let process = AsyncProcess(
-                arguments: [self.tarCommand, "acf", destinationPath.pathString, directory.basename],
-                environment: .current,
-                workingDirectory: directory.parentDirectory
-            )
+        guard self.fileSystem.isDirectory(directory) else {
+            throw FileSystemError(.notDirectory, directory.underlying)
+        }
 
-            guard let registrationKey = self.cancellator.register(process) else {
-                throw CancellationError.failedToRegisterProcess(process)
-            }
+        let process = AsyncProcess(
+            arguments: [self.tarCommand, "acf", destinationPath.pathString, directory.basename],
+            environment: .current,
+            workingDirectory: directory.parentDirectory
+        )
 
-            DispatchQueue.sharedConcurrent.async {
-                defer { self.cancellator.deregister(registrationKey) }
-                completion(.init(catching: {
-                    try process.launch()
-                    let processResult = try process.waitUntilExit()
-                    guard processResult.exitStatus == .terminated(code: 0) else {
-                        throw try StringError(processResult.utf8stderrOutput())
-                    }
-                }))
-            }
-        } catch {
-            return completion(.failure(error))
+        guard let registrationKey = self.cancellator.register(process) else {
+            throw CancellationError.failedToRegisterProcess(process)
+        }
+
+        defer { self.cancellator.deregister(registrationKey) }
+
+        try process.launch()
+        let processResult = try await process.waitUntilExit()
+        guard processResult.exitStatus == .terminated(code: 0) else {
+            throw try StringError(processResult.utf8stderrOutput())
         }
     }
 

Sources/Basics/Archiver/UniversalArchiver.swift

@@ -85,15 +85,10 @@ public struct UniversalArchiver: Archiver {
 
     public func compress(
         directory: AbsolutePath,
-        to destinationPath: AbsolutePath,
-        completion: @escaping @Sendable (Result<Void, Swift.Error>) -> Void
-    ) {
-        do {
-            let archiver = try archiver(for: destinationPath)
-            archiver.compress(directory: directory, to: destinationPath, completion: completion)
-        } catch {
-            completion(.failure(error))
-        }
+        to destinationPath: AbsolutePath
+    ) async throws {
+        let archiver = try archiver(for: destinationPath)
+        try await archiver.compress(directory: directory, to: destinationPath)
     }
 
     public func validate(

Sources/Basics/Archiver/ZipArchiver.swift

@@ -73,52 +73,44 @@ public struct ZipArchiver: Archiver, Cancellable {
 
     public func compress(
         directory: AbsolutePath,
-        to destinationPath: AbsolutePath,
-        completion: @escaping @Sendable (Result<Void, Error>) -> Void
-    ) {
-        do {
-            guard self.fileSystem.isDirectory(directory) else {
-                throw FileSystemError(.notDirectory, directory.underlying)
-            }
+        to destinationPath: AbsolutePath
+    ) async throws {
+        guard self.fileSystem.isDirectory(directory) else {
+            throw FileSystemError(.notDirectory, directory.underlying)
+        }
 
-            #if os(Windows)
-            let process = AsyncProcess(
-                // FIXME: are these the right arguments?
-                arguments: ["tar.exe", "-a", "-c", "-f", destinationPath.pathString, directory.basename],
-                workingDirectory: directory.parentDirectory
-            )
-            #else
-            // This is to work around `swift package-registry publish` tool failing on
-            // Amazon Linux 2 due to it having an earlier Glibc version (rdar://116370323)
-            // and therefore posix_spawn_file_actions_addchdir_np is unavailable.
-            // Instead of passing `workingDirectory` param to TSC.Process, which will trigger
-            // SPM_posix_spawn_file_actions_addchdir_np_supported check, we shell out and
-            // do `cd` explicitly before `zip`.
-            let process = AsyncProcess(
-                arguments: [
-                    "/bin/sh",
-                    "-c",
-                    "cd \(directory.parentDirectory.underlying.pathString) && zip -r \(destinationPath.pathString) \(directory.basename)",
-                ]
-            )
-            #endif
+        #if os(Windows)
+        let process = AsyncProcess(
+            // FIXME: are these the right arguments?
+            arguments: ["tar.exe", "-a", "-c", "-f", destinationPath.pathString, directory.basename],
+            workingDirectory: directory.parentDirectory
+        )
+        #else
+        // This is to work around `swift package-registry publish` tool failing on
+        // Amazon Linux 2 due to it having an earlier Glibc version (rdar://116370323)
+        // and therefore posix_spawn_file_actions_addchdir_np is unavailable.
+        // Instead of passing `workingDirectory` param to TSC.Process, which will trigger
+        // SPM_posix_spawn_file_actions_addchdir_np_supported check, we shell out and
+        // do `cd` explicitly before `zip`.
+        let process = AsyncProcess(
+            arguments: [
+                "/bin/sh",
+                "-c",
+                "cd \(directory.parentDirectory.underlying.pathString) && zip -r \(destinationPath.pathString) \(directory.basename)",
+            ]
+        )
+        #endif
+
+        guard let registrationKey = self.cancellator.register(process) else {
+            throw CancellationError.failedToRegisterProcess(process)
+        }
 
-            guard let registrationKey = self.cancellator.register(process) else {
-                throw CancellationError.failedToRegisterProcess(process)
-            }
+        defer { self.cancellator.deregister(registrationKey) }
 
-            DispatchQueue.sharedConcurrent.async {
-                defer { self.cancellator.deregister(registrationKey) }
-                completion(.init(catching: {
-                    try process.launch()
-                    let processResult = try process.waitUntilExit()
-                    guard processResult.exitStatus == .terminated(code: 0) else {
-                        throw try StringError(processResult.utf8stderrOutput())
-                    }
-                }))
-            }
-        } catch {
-            return completion(.failure(error))
+        try process.launch()
+        let processResult = try await process.waitUntilExit()
+        guard processResult.exitStatus == .terminated(code: 0) else {
+            throw try StringError(processResult.utf8stderrOutput())
         }
     }
 

Sources/Basics/AuthorizationProvider.swift

@@ -22,41 +22,14 @@ public protocol AuthorizationProvider: Sendable {
 }
 
 public protocol AuthorizationWriter {
-    @available(*, noasync, message: "Use the async alternative")
     func addOrUpdate(
         for url: URL,
         user: String,
         password: String,
-        persist: Bool,
-        callback: @escaping (Result<Void, Error>) -> Void
-    )
+        persist: Bool
+    ) async throws
 
-    @available(*, noasync, message: "Use the async alternative")
-    func remove(for url: URL, callback: @escaping (Result<Void, Error>) -> Void)
-}
-
-public extension AuthorizationWriter {
-    func addOrUpdate(
-        for url: URL,
-        user: String,
-        password: String,
-        persist: Bool = true
-    ) async throws {
-        try await safe_async {
-            self.addOrUpdate(
-                for: url,
-                user: user,
-                password: password, 
-                persist: persist,
-                callback: $0)
-        }
-    }
-
-    func remove(for url: URL) async throws {
-        try await safe_async {
-            self.remove(for: url, callback: $0)
-        }
-    }
+    func remove(for url: URL) async throws
 }
 
 public enum AuthorizationProviderError: Error {
@@ -100,24 +73,23 @@ public final class NetrcAuthorizationProvider: AuthorizationProvider, Authorizat
         for url: URL,
         user: String,
         password: String,
-        persist: Bool = true,
-        callback: @escaping (Result<Void, Error>) -> Void
-    ) {
+        persist: Bool = true
+    ) async throws {
         guard let machine = Self.machine(for: url) else {
-            return callback(.failure(AuthorizationProviderError.invalidURLHost))
+            throw AuthorizationProviderError.invalidURLHost
         }
 
         if !persist {
             self.cache[machine] = (user, password)
-            return callback(.success(()))
+            return
         }
 
         // Same entry already exists, no need to add or update
         let netrc = try? Self.load(fileSystem: self.fileSystem, path: self.path)
         guard netrc?.machines
             .first(where: { $0.name.lowercased() == machine && $0.login == user && $0.password == password }) == nil
         else {
-            return callback(.success(()))
+            return
         }
 
         do {
@@ -134,21 +106,15 @@ public final class NetrcAuthorizationProvider: AuthorizationProvider, Authorizat
                     stream.write("\n")
                 }
             }
-
-            callback(.success(()))
         } catch {
-            callback(.failure(
-                AuthorizationProviderError
-                    .other("Failed to update netrc file at \(self.path): \(error.interpolationDescription)")
-            ))
+            throw AuthorizationProviderError
+                .other("Failed to update netrc file at \(self.path): \(error.interpolationDescription)")
         }
     }
 
-    public func remove(for url: URL, callback: @escaping (Result<Void, Error>) -> Void) {
-        callback(.failure(
-            AuthorizationProviderError
-                .other("User must edit netrc file at \(self.path) manually to remove entries")
-        ))
+    public func remove(for url: URL) async throws {
+        throw AuthorizationProviderError
+            .other("User must edit netrc file at \(self.path) manually to remove entries")
     }
 
     public func authentication(for url: URL) -> (user: String, password: String)? {
@@ -217,47 +183,36 @@ public final class KeychainAuthorizationProvider: AuthorizationProvider, Authori
         for url: URL,
         user: String,
         password: String,
-        persist: Bool = true,
-        callback: @escaping (Result<Void, Error>) -> Void
-    ) {
+        persist: Bool = true
+    ) async throws {
         guard let protocolHostPort = ProtocolHostPort(from: url) else {
-            return callback(.failure(AuthorizationProviderError.invalidURLHost))
+            throw AuthorizationProviderError.invalidURLHost
         }
 
         self.observabilityScope
             .emit(debug: "add/update credentials for '\(protocolHostPort)' [\(url.absoluteString)] in keychain")
 
         if !persist {
             self.cache[protocolHostPort.description] = (user, password)
-            return callback(.success(()))
+            return
         }
 
         let passwordData = Data(password.utf8)
 
-        do {
-            if !(try self.update(protocolHostPort: protocolHostPort, account: user, password: passwordData)) {
-                try self.create(protocolHostPort: protocolHostPort, account: user, password: passwordData)
-            }
-            callback(.success(()))
-        } catch {
-            callback(.failure(error))
+        if !(try self.update(protocolHostPort: protocolHostPort, account: user, password: passwordData)) {
+            try self.create(protocolHostPort: protocolHostPort, account: user, password: passwordData)
         }
     }
 
-    public func remove(for url: URL, callback: @escaping (Result<Void, Error>) -> Void) {
+    public func remove(for url: URL) async throws {
         guard let protocolHostPort = ProtocolHostPort(from: url) else {
-            return callback(.failure(AuthorizationProviderError.invalidURLHost))
+            throw AuthorizationProviderError.invalidURLHost
         }
 
         self.observabilityScope
             .emit(debug: "remove credentials for '\(protocolHostPort)' [\(url.absoluteString)] from keychain")
 
-        do {
-            try self.delete(protocolHostPort: protocolHostPort)
-            callback(.success(()))
-        } catch {
-            callback(.failure(error))
-        }
+        try self.delete(protocolHostPort: protocolHostPort)
     }
 
     public func authentication(for url: URL) -> (user: String, password: String)? {