Make modules graph loading async

Sources/Basics/Concurrency/ThreadSafeBox.swift

@@ -43,6 +43,18 @@ public final class ThreadSafeBox<Value> {
         return value
     }
 
+    @discardableResult
+    public func memoize(body: () async throws -> Value) async rethrows -> Value {
+        if let value = self.get() {
+            return value
+        }
+        let value = try await body()
+        self.lock.withLock {
+            self.underlying = value
+        }
+        return value
+    }
+
     public func clear() {
         self.lock.withLock {
             self.underlying = nil

Sources/Basics/FileSystem/AbsolutePath.swift

@@ -40,7 +40,7 @@ public struct AbsolutePath: Hashable, Sendable {
     /// Root directory (whose string representation is just a path separator).
     public static let root = Self(TSCAbsolutePath.root)
 
-    internal let underlying: TSCAbsolutePath
+    package let underlying: TSCAbsolutePath
 
     // public for transition
     public init(_ underlying: TSCAbsolutePath) {

Sources/Build/BuildOperation.swift

@@ -144,7 +144,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     private let config: LLBuildSystemConfiguration
 
     /// The closure for loading the package graph.
-    let packageGraphLoader: () throws -> ModulesGraph
+    let packageGraphLoader: () async throws -> ModulesGraph
 
     /// the plugin configuration for build plugins
     let pluginConfiguration: PluginConfiguration?
@@ -186,7 +186,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 
     public var builtTestProducts: [BuiltTestProduct] {
-        (try? getBuildDescription())?.builtTestProducts ?? []
+        get async {
+            (try? await getBuildDescription())?.builtTestProducts ?? []
+        }
     }
 
     /// File rules to determine resource handling behavior.
@@ -205,7 +207,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         productsBuildParameters: BuildParameters,
         toolsBuildParameters: BuildParameters,
         cacheBuildManifest: Bool,
-        packageGraphLoader: @escaping () throws -> ModulesGraph,
+        packageGraphLoader: @escaping () async throws -> ModulesGraph,
         pluginConfiguration: PluginConfiguration? = .none,
         scratchDirectory: AbsolutePath,
         additionalFileRules: [FileRuleDescription],
@@ -280,18 +282,20 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.rootPackageIdentityByTargetName = (try? Dictionary<String, PackageIdentity>(throwingUniqueKeysWithValues: targetsByRootPackageIdentity.lazy.flatMap { e in e.value.map { ($0, e.key) } })) ?? [:]
     }
 
-    public func getPackageGraph() throws -> ModulesGraph {
-        try self.packageGraph.memoize {
-            try self.packageGraphLoader()
+    public var modulesGraph: ModulesGraph {
+        get async throws {
+            try await self.packageGraph.memoize {
+                try await self.packageGraphLoader()
+            }
         }
     }
 
     /// Compute and return the latest build description.
     ///
     /// This will try skip build planning if build manifest caching is enabled
     /// and the package structure hasn't changed.
-    public func getBuildDescription(subset: BuildSubset? = nil) throws -> BuildDescription {
-        return try self.buildDescription.memoize {
+    public func getBuildDescription(subset: BuildSubset? = nil) async throws -> BuildDescription {
+        return try await self.buildDescription.memoize {
             if self.cacheBuildManifest {
                 do {
                     // if buildPackageStructure returns a valid description we use that, otherwise we perform full planning
@@ -320,12 +324,12 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 }
             }
             // We need to perform actual planning if we reach here.
-            return try self.plan(subset: subset).description
+            return try await self.plan(subset: subset).description
         }
     }
 
-    public func getBuildManifest() throws -> LLBuildManifest {
-        return try self.plan().manifest
+    public func getBuildManifest() async throws -> LLBuildManifest {
+        return try await self.plan().manifest
     }
 
     /// Cancel the active build operation.
@@ -475,7 +479,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 
     /// Perform a build using the given build description and subset.
-    public func build(subset: BuildSubset) throws {
+    public func build(subset: BuildSubset) async throws {
         guard !self.config.shouldSkipBuilding(for: .target) else {
             return
         }
@@ -485,7 +489,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         // Get the build description (either a cached one or newly created).
 
         // Get the build description
-        let buildDescription = try getBuildDescription(subset: subset)
+        let buildDescription = try await getBuildDescription(subset: subset)
 
         // Verify dependency imports on the described targets
         try verifyTargetImports(in: buildDescription)
@@ -500,7 +504,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         // If any plugins are part of the build set, compile them now to surface
         // any errors up-front. Returns true if we should proceed with the build
         // or false if not. It will already have thrown any appropriate error.
-        guard try self.compilePlugins(in: subset) else {
+        guard try await self.compilePlugins(in: subset) else {
             return
         }
 
@@ -509,7 +513,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         progressTracker.buildStart(configuration: configuration)
 
         // Perform the build.
-        let llbuildTarget = try computeLLBuildTargetName(for: subset)
+        let llbuildTarget = try await computeLLBuildTargetName(for: subset)
         let success = buildSystem.build(target: llbuildTarget)
 
         let duration = buildStartTime.distance(to: .now())
@@ -566,10 +570,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// true if the build should proceed. Throws an error in case of failure. A
     /// reason why the build might not proceed even on success is if only plugins
     /// should be compiled.
-    func compilePlugins(in subset: BuildSubset) throws -> Bool {
+    func compilePlugins(in subset: BuildSubset) async throws -> Bool {
         // Figure out what, if any, plugin descriptions to compile, and whether
         // to continue building after that based on the subset.
-        let allPlugins = try getBuildDescription().pluginDescriptions
+        let allPlugins = try await getBuildDescription().pluginDescriptions
         let pluginsToCompile: [PluginBuildDescription]
         let continueBuilding: Bool
         switch subset {
@@ -662,15 +666,15 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 
     /// Compute the llbuild target name using the given subset.
-    func computeLLBuildTargetName(for subset: BuildSubset) throws -> String {
+    func computeLLBuildTargetName(for subset: BuildSubset) async throws -> String {
         switch subset {
         case .allExcludingTests:
             return LLBuildManifestBuilder.TargetKind.main.targetName
         case .allIncludingTests:
             return LLBuildManifestBuilder.TargetKind.test.targetName
         case .product(let productName, let destination):
             // FIXME: This is super unfortunate that we might need to load the package graph.
-            let graph = try getPackageGraph()
+            let graph = try await self.modulesGraph
 
             let buildTriple: BuildTriple? = if let destination {
                 destination == .host ? .tools : .destination
@@ -704,7 +708,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             return try product.getLLBuildTargetName(buildParameters: buildParameters)
         case .target(let targetName, let destination):
             // FIXME: This is super unfortunate that we might need to load the package graph.
-            let graph = try getPackageGraph()
+            let graph = try await self.modulesGraph
 
             let buildTriple: BuildTriple? = if let destination {
                 destination == .host ? .tools : .destination
@@ -731,9 +735,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 
     /// Create the build plan and return the build description.
-    private func plan(subset: BuildSubset? = nil) throws -> (description: BuildDescription, manifest: LLBuildManifest) {
+    private func plan(subset: BuildSubset? = nil) async throws -> BuildManifestDescription {
         // Load the package graph.
-        let graph = try getPackageGraph()
+        let graph = try await self.modulesGraph
         let buildToolPluginInvocationResults: [ResolvedModule.ID: (target: ResolvedModule, results: [BuildToolPluginInvocationResult])]
         let prebuildCommandResults: [ResolvedModule.ID: [PrebuildCommandResult]]
         // Invoke any build tool plugins in the graph to generate prebuild commands and build commands.
@@ -748,7 +752,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 hostTriple: try pluginConfiguration.scriptRunner.hostTriple
             )
 
-            buildToolPluginInvocationResults = try graph.invokeBuildToolPlugins(
+            buildToolPluginInvocationResults = try await graph.invokeBuildToolPlugins(
                 pluginsPerTarget: pluginsPerModule,
                 pluginTools: pluginTools,
                 outputDir: pluginConfiguration.workDirectory.appending("outputs"),
@@ -859,7 +863,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         )
 
         // Finally create the llbuild manifest from the plan.
-        return (buildDescription, buildManifest)
+        return .init(description: buildDescription, manifest: buildManifest)
     }
 
     /// Build the package structure target.
@@ -987,7 +991,16 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     public func packageStructureChanged() -> Bool {
         do {
-            _ = try self.plan()
+            _ = try temp_await { (callback: @escaping (Result<BuildManifestDescription, any Error>) -> Void) in
+                _Concurrency.Task {
+                    do {
+                        let value = try await self.plan()
+                        callback(.success(value))
+                    } catch {
+                        callback(.failure(error))
+                    }
+                }
+            }
         }
         catch Diagnostics.fatalError {
             return false

Sources/Commands/PackageCommands/APIDiff.swift

@@ -32,7 +32,7 @@ struct DeprecatedAPIDiff: ParsableCommand {
     }
 }
 
-struct APIDiff: SwiftCommand {
+struct APIDiff: AsyncSwiftCommand {
     static let configuration = CommandConfiguration(
         commandName: "diagnose-api-breaking-changes",
         abstract: "Diagnose API-breaking changes to Swift modules in a package",
@@ -77,7 +77,7 @@ struct APIDiff: SwiftCommand {
     @Flag(help: "Regenerate the API baseline, even if an existing one is available.")
     var regenerateBaseline: Bool = false
 
-    func run(_ swiftCommandState: SwiftCommandState) throws {
+    func run(_ swiftCommandState: SwiftCommandState) async throws {
         let apiDigesterPath = try swiftCommandState.getTargetToolchain().getSwiftAPIDigester()
         let apiDigesterTool = SwiftAPIDigester(fileSystem: swiftCommandState.fileSystem, tool: apiDigesterPath)
 
@@ -92,14 +92,14 @@ struct APIDiff: SwiftCommand {
             cacheBuildManifest: false
         )
 
-        let packageGraph = try buildSystem.getPackageGraph()
+        let packageGraph = try await buildSystem.modulesGraph
         let modulesToDiff = try determineModulesToDiff(
             packageGraph: packageGraph,
             observabilityScope: swiftCommandState.observabilityScope
         )
 
         // Build the current package.
-        try buildSystem.build()
+        try await buildSystem.build()
 
         // Dump JSON for the baseline package.
         let baselineDumper = try APIDigesterBaselineDumper(
@@ -111,7 +111,7 @@ struct APIDiff: SwiftCommand {
             observabilityScope: swiftCommandState.observabilityScope
         )
 
-        let baselineDir = try baselineDumper.emitAPIBaseline(
+        let baselineDir = try await baselineDumper.emitAPIBaseline(
             for: modulesToDiff,
             at: overrideBaselineDir,
             force: regenerateBaseline,

Sources/Commands/PackageCommands/DumpCommands.swift

@@ -17,7 +17,7 @@ import Foundation
 import PackageModel
 import XCBuildSupport
 
-struct DumpSymbolGraph: SwiftCommand {
+struct DumpSymbolGraph: AsyncSwiftCommand {
     static let configuration = CommandConfiguration(
         abstract: "Dump Symbol Graph")
     static let defaultMinimumAccessLevel = SymbolGraphExtract.AccessLevel.public
@@ -43,7 +43,7 @@ struct DumpSymbolGraph: SwiftCommand {
     @Flag(help: "Emit extension block symbols for extensions to external types or directly associate members and conformances with the extended nominal.")
     var extensionBlockSymbolBehavior: ExtensionBlockSymbolBehavior = .omitExtensionBlockSymbols
 
-    func run(_ swiftCommandState: SwiftCommandState) throws {
+    func run(_ swiftCommandState: SwiftCommandState) async throws {
         // Build the current package.
         //
         // We turn build manifest caching off because we need the build plan.
@@ -53,7 +53,7 @@ struct DumpSymbolGraph: SwiftCommand {
             traitConfiguration: .init(enableAllTraits: true),
             cacheBuildManifest: false
         )
-        try buildSystem.build()
+        try await buildSystem.build()
 
         // Configure the symbol graph extractor.
         let symbolGraphExtractor = try SymbolGraphExtract(
@@ -71,7 +71,7 @@ struct DumpSymbolGraph: SwiftCommand {
         // Run the tool once for every library and executable target in the root package.
         let buildPlan = try buildSystem.buildPlan
         let symbolGraphDirectory = buildPlan.destinationBuildParameters.dataPath.appending("symbolgraph")
-        let targets = try buildSystem.getPackageGraph().rootPackages.flatMap{ $0.modules }.filter{ $0.type == .library }
+        let targets = try await buildSystem.getPackageGraph().rootPackages.flatMap{ $0.modules }.filter{ $0.type == .library }
         for target in targets {
             print("-- Emitting symbol graph for", target.name)
             let result = try symbolGraphExtractor.extractSymbolGraph(

Sources/Commands/PackageCommands/InstalledPackages.swift

@@ -18,7 +18,7 @@ import PackageModel
 import TSCBasic
 
 extension SwiftPackageCommand {
-    struct Install: SwiftCommand {
+    struct Install: AsyncSwiftCommand {
         static let configuration = CommandConfiguration(
             commandName: "experimental-install",
             abstract: "Offers the ability to install executable products of the current package."
@@ -30,7 +30,7 @@ extension SwiftPackageCommand {
         @Option(help: "The name of the executable product to install")
         var product: String?
 
-        func run(_ tool: SwiftCommandState) throws {
+        func run(_ tool: SwiftCommandState) async throws {
             let swiftpmBinDir = try tool.fileSystem.getOrCreateSwiftPMInstalledBinariesDirectory()
 
             let env = Environment.current
@@ -81,7 +81,7 @@ extension SwiftPackageCommand {
                 throw StringError("\(productToInstall.name) is already installed at \(existingPkg.path)")
             }
 
-            try tool.createBuildSystem(explicitProduct: productToInstall.name, traitConfiguration: .init())
+            try await tool.createBuildSystem(explicitProduct: productToInstall.name, traitConfiguration: .init())
                 .build(subset: .product(productToInstall.name))
 
             let binPath = try tool.productsBuildParameters.buildPath.appending(component: productToInstall.name)

Sources/Commands/PackageCommands/Learn.swift

@@ -17,8 +17,7 @@ import PackageGraph
 import PackageModel
 
 extension SwiftPackageCommand {
-    struct Learn: SwiftCommand {
-
+    struct Learn: AsyncSwiftCommand {
         @OptionGroup()
         var globalOptions: GlobalOptions
 
@@ -90,7 +89,7 @@ extension SwiftPackageCommand {
             return snippetGroups.filter { !$0.snippets.isEmpty }
         }
 
-        func run(_ swiftCommandState: SwiftCommandState) throws {
+        func run(_ swiftCommandState: SwiftCommandState) async throws {
             let graph = try swiftCommandState.loadPackageGraph()
             let package = graph.rootPackages[graph.rootPackages.startIndex]
             print(package.products.map { $0.description })
@@ -99,7 +98,7 @@ extension SwiftPackageCommand {
 
             var cardStack = CardStack(package: package, snippetGroups: snippetGroups, swiftCommandState: swiftCommandState)
 
-            cardStack.run()
+            await cardStack.run()
         }
     }
 }