[Dependency Scanning] Move computation of the path for libSwiftScan to the toolchain

Alongside lookup of executable tools, the toolchain is now responsible for looking up the driver's compiler support and dependency scanning library.

It also adds an environment variable override 'SWIFT_DRIVER_SWIFTSCAN_LIB' for clients to specify a specific path to this shared library that they would like to be used instead of the default.

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -650,11 +650,10 @@ public struct Driver {
                                                                                outputFileMap: outputFileMap)
 
     self.supportedFrontendFlags =
-      try Self.computeSupportedCompilerArgs(of: self.toolchain, hostTriple: self.hostTriple,
-                                                parsedOptions: &self.parsedOptions,
-                                                diagnosticsEngine: diagnosticEngine,
-                                                fileSystem: fileSystem, executor: executor,
-                                                env: env)
+      try Self.computeSupportedCompilerArgs(of: self.toolchain,
+                                            parsedOptions: &self.parsedOptions,
+                                            diagnosticsEngine: diagnosticEngine,
+                                            executor: executor)
     let supportedFrontendFlagsLocal = self.supportedFrontendFlags
     self.savedUnknownDriverFlagsForSwiftFrontend = try self.parsedOptions.saveUnknownFlags {
       Driver.isOptionFound($0, allOpts: supportedFrontendFlagsLocal)

Sources/SwiftDriver/Driver/WindowsExtensions.swift

@@ -20,3 +20,27 @@ internal func executableName(_ name: String) -> String {
   return name
 #endif
 }
+
+@_spi(Testing) public func sharedLibraryName(_ name: String) -> String {
+#if canImport(Darwin)
+  let ext = ".dylib"
+#elseif os(Windows)
+  let ext = ".dll"
+#else
+  let ext = ".so"
+#endif
+  return name + ext
+}
+
+// FIXME: This can be subtly wrong, we should rather
+// try to get the client to provide this info or move to a better
+// path convention for where we keep compiler support libraries
+internal var compilerHostSupportLibraryOSComponent : String {
+#if canImport(Darwin)
+  return "macosx"
+#elseif os(Windows)
+  return "windows"
+#else
+  return "linux"
+#endif
+}

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -221,7 +221,7 @@ public extension Driver {
     // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency
     // scanning.
     var fallbackToFrontend = parsedOptions.hasArgument(.driverScanDependenciesNonLib)
-    let scanLibPath = try Self.getScanLibPath(of: toolchain, hostTriple: hostTriple, env: env)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     if try interModuleDependencyOracle
         .verifyOrCreateScannerInstance(fileSystem: fileSystem,
                                        swiftScanLibPath: scanLibPath) == false {
@@ -490,46 +490,9 @@ public extension Driver {
                                                      useResponseFiles: useResponseFiles)
     return args
   }
-}
-
-@_spi(Testing) public extension Driver {
-  static func getScanLibPath(of toolchain: Toolchain, hostTriple: Triple,
-                             env: [String: String]) throws -> AbsolutePath {
-    if hostTriple.isWindows {
-      // no matter if we are in a build tree or an installed tree, the layout is
-      // always: `bin/_InternalSwiftScan.dll`
-      return try getRootPath(of: toolchain, env: env)
-                    .appending(component: "bin")
-                    .appending(component: "_InternalSwiftScan.dll")
-    }
-
-    let sharedLibExt: String
-    if hostTriple.isMacOSX {
-      sharedLibExt = ".dylib"
-    } else {
-      sharedLibExt = ".so"
-    }
-    let libScanner = "lib_InternalSwiftScan\(sharedLibExt)"
-    // We first look into position in toolchain
-    let libPath
-     = try getRootPath(of: toolchain, env: env).appending(component: "lib")
-      .appending(component: "swift")
-      .appending(component: hostTriple.osNameUnversioned)
-      .appending(component: libScanner)
-    if localFileSystem.exists(libPath) {
-        return libPath
-    }
-    // In case we are using a compiler from the build dir, we should also try
-    // this path.
-    return try getRootPath(of: toolchain, env: env).appending(component: "lib")
-      .appending(component: libScanner)
-  }
 
   static func getRootPath(of toolchain: Toolchain, env: [String: String])
   throws -> AbsolutePath {
-    if let overrideString = env["SWIFT_DRIVER_SWIFT_SCAN_TOOLCHAIN_PATH"] {
-      return try AbsolutePath(validating: overrideString)
-    }
     return try toolchain.getToolPath(.swiftCompiler)
       .parentDirectory // bin
       .parentDirectory // toolchain root

Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift

@@ -57,6 +57,7 @@ extension Toolchain {
 }
 
 extension Driver {
+<<<<<<< HEAD
   static func computeSupportedCompilerArgs(of toolchain: Toolchain, hostTriple: Triple,
                                                parsedOptions: inout ParsedOptions,
                                                diagnosticsEngine: DiagnosticsEngine,

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -317,14 +317,10 @@ internal final class SwiftScan {
 @_spi(Testing) public extension Driver {
   func querySupportedArgumentsForTest() throws -> Set<String>? {
     // If a capable libSwiftScan is found, manually ensure we can get the supported arguments
-    let scanLibPath = try Self.getScanLibPath(of: toolchain,
-                                              hostTriple: hostTriple,
-                                              env: env)
-    if fileSystem.exists(scanLibPath) {
-      let libSwiftScanInstance = try SwiftScan(dylib: scanLibPath)
-      if libSwiftScanInstance.canQuerySupportedArguments() {
-        return try libSwiftScanInstance.querySupportedArguments()
-      }
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
+    let libSwiftScanInstance = try SwiftScan(dylib: scanLibPath)
+    if libSwiftScanInstance.canQuerySupportedArguments() {
+      return try libSwiftScanInstance.querySupportedArguments()
     }
     return nil
   }

Sources/SwiftDriver/Toolchains/Toolchain.swift

@@ -245,6 +245,46 @@ extension Toolchain {
     }
   }
 
+  /// Looks for the executable in the `SWIFT_DRIVER_SWIFTSCAN_LIB` environment variable, if found nothing,
+  /// looks in the `lib` relative to the compiler executable.
+  /// TODO: If the driver needs to lookup other shared libraries, this is simple to generalize
+  @_spi(Testing) public func lookupSwiftScanLib() throws -> AbsolutePath {
+#if os(Windows)
+    // no matter if we are in a build tree or an installed tree, the layout is
+    // always: `bin/_InternalSwiftScan.dll`
+    return try toolchain.getToolPath(.swiftCompiler)
+                        .parentDirectory // bin
+                        .appending(component: "_InternalSwiftScan.dll")
+#else
+    let libraryName = sharedLibraryName("lib_InternalSwiftScan")
+    if let overrideString = env["SWIFT_DRIVER_SWIFTSCAN_LIB"],
+       let path = try? AbsolutePath(validating: overrideString) {
+      return path
+    } else {
+      let compilerPath = try getToolPath(.swiftCompiler)
+      let toolchainRootPath = compilerPath.parentDirectory // bin
+                                          .parentDirectory // toolchain root
+
+      let searchPaths = [toolchainRootPath.appending(component: "lib")
+                                          .appending(component: "swift")
+                                          .appending(component: compilerHostSupportLibraryOSComponent),
+                         toolchainRootPath.appending(component: "lib")
+                                          .appending(component: "swift")
+                                          .appending(component: "host"),
+                         // In case we are using a compiler from the build dir, we should also try
+                         // this path.
+                         toolchainRootPath.appending(component: "lib")]
+      for libraryPath in searchPaths.map({ $0.appending(component: libraryName) }) {
+        if fileSystem.isFile(libraryPath) {
+          return libraryPath
+        }
+      }
+    }
+
+    throw ToolchainError.unableToFind(tool: libraryName)
+#endif
+  }
+
   private func xcrunFind(executable: String) throws -> AbsolutePath {
     let xcrun = "xcrun"
     guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -640,7 +640,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testModuleAliasingWithImportPrescan() throws {
-    let (_, _, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (_, _, toolchain, _) = try getDriverArtifactsForScanning()
 
     let dummyDriver = try Driver(args: ["swiftc", "-module-name", "dummyDriverCheck", "test.swift"])
     guard dummyDriver.isFrontendArgSupported(.moduleAlias) else {
@@ -650,9 +650,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -832,9 +830,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
       // 2. Run a dependency scan to find the just-built module
       let dependencyOracle = InterModuleDependencyOracle()
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       guard try dependencyOracle
               .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                              swiftScanLibPath: scanLibPath) else {
@@ -931,14 +927,12 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
   /// Test the libSwiftScan dependency scanning (import-prescan).
   func testDependencyImportPrescan() throws {
-    let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdLibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
 
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -1009,14 +1003,12 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testDependencyScanningFailure() throws {
-    let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
 
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
       .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                      swiftScanLibPath: scanLibPath) else {
@@ -1087,9 +1079,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
     // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
     // queries.
     let dependencyOracle = InterModuleDependencyOracle()
-    let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                hostTriple: hostTriple,
-                                                env: ProcessEnv.vars)
+    let scanLibPath = try toolchain.lookupSwiftScanLib()
     guard try dependencyOracle
             .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                            swiftScanLibPath: scanLibPath) else {
@@ -1277,11 +1267,9 @@ final class ExplicitModuleBuildTests: XCTestCase {
   }
 
   func testDependencyGraphDotSerialization() throws {
-      let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+      let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
       let dependencyOracle = InterModuleDependencyOracle()
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       guard try dependencyOracle
               .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
                                              swiftScanLibPath: scanLibPath) else {
@@ -1342,7 +1330,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
 
   /// Test the libSwiftScan dependency scanning.
   func testDependencyScanReuseCache() throws {
-    let (stdlibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
+    let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning()
     try withTemporaryDirectory { path in
       let cacheSavePath = path.appending(component: "saved.moddepcache")
       let main = path.appending(component: "testDependencyScanning.swift")
@@ -1378,9 +1366,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
         scannerCommand.removeFirst()
       }
 
-      let scanLibPath = try Driver.getScanLibPath(of: toolchain,
-                                                  hostTriple: hostTriple,
-                                                  env: ProcessEnv.vars)
+      let scanLibPath = try toolchain.lookupSwiftScanLib()
       // Run the first scan and serialize the cache contents.
       let firstDependencyOracle = InterModuleDependencyOracle()
       guard try firstDependencyOracle

Tests/SwiftDriverTests/SwiftDriverTests.swift

@@ -6685,6 +6685,8 @@ final class SwiftDriverTests: XCTestCase {
     let PATH = "PATH"
 #endif
     let SWIFT_FRONTEND_EXEC = "SWIFT_DRIVER_SWIFT_FRONTEND_EXEC"
+    let SWIFT_SCANNER_LIB = "SWIFT_DRIVER_SWIFTSCAN_LIB"
+
 
     // Reset the environment to ensure tool resolution is exactly run against PATH.
     var driver = try Driver(args: ["swiftc", "-print-target-info"], env: [PATH: ProcessEnv.path!])
@@ -6695,6 +6697,7 @@ final class SwiftDriverTests: XCTestCase {
 
     try withTemporaryDirectory { toolsDirectory in
       let customSwiftFrontend = toolsDirectory.appending(component: executableName("swift-frontend"))
+      let customSwiftScan = toolsDirectory.appending(component: sharedLibraryName("lib_InternalSwiftScan"))
       try localFileSystem.createSymbolicLink(customSwiftFrontend, pointingAt: defaultSwiftFrontend, relative: false)
 
       try withTemporaryDirectory { tempDirectory in 
@@ -6707,7 +6710,9 @@ final class SwiftDriverTests: XCTestCase {
         // test if SWIFT_DRIVER_TOOLNAME_EXEC is respected
         do {
           var driver = try Driver(args: ["swiftc", "-print-target-info"],
-                                  env: [PATH: ProcessEnv.path!, SWIFT_FRONTEND_EXEC: customSwiftFrontend.pathString])
+                                  env: [PATH: ProcessEnv.path!,
+                                        SWIFT_FRONTEND_EXEC: customSwiftFrontend.pathString,
+                                        SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, customSwiftFrontend.pathString)
@@ -6716,15 +6721,16 @@ final class SwiftDriverTests: XCTestCase {
         // test if tools directory is respected
         do {
           var driver = try Driver(args: ["swiftc", "-print-target-info", "-tools-directory", toolsDirectory.pathString],
-                                  env: [PATH: ProcessEnv.path!])
+                                  env: [PATH: ProcessEnv.path!, SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, customSwiftFrontend.pathString)
         }
 
         // test if current working directory is searched before PATH
         do {
-          var driver = try Driver(args: ["swiftc", "-print-target-info"], env: [PATH: toolsDirectory.pathString])
+          var driver = try Driver(args: ["swiftc", "-print-target-info"],
+                                  env: [PATH: toolsDirectory.pathString, SWIFT_SCANNER_LIB: customSwiftScan.pathString])
           let jobs = try driver.planBuild()
           XCTAssertEqual(jobs.count, 1)
           XCTAssertEqual(jobs.first!.tool.name, anotherSwiftFrontend.pathString)