[Explicit Module Builds] Add support for header dependencies of binary Swift module dependencies

When we encounter a pre-built Swift binary module dependency (without an interface file), such module may have been built with a bridging header, which must still be present and is referenced by the binary .swiftmodule as either a .h or, more-likely, a pre-built .pch in a fully-explicit build.

Clients must be able to know about such header dependencies in order to be able to import this binary module, because this binary module may be referencing types brought in via its bridging header. The build-system client (swift-driver) will then ensure these header dependencies are fed as inputs to all requiring compilation tasks.

This adds support to the driver to query such header dependencies and feed them as inputs to all requiring compilation tasks.

Author: artemcm

Sources/CSwiftScan/include/swiftscan_header.h

@@ -132,6 +132,8 @@ typedef struct {
   (*swiftscan_swift_binary_detail_get_module_doc_path)(swiftscan_module_details_t);
   swiftscan_string_ref_t
   (*swiftscan_swift_binary_detail_get_module_source_info_path)(swiftscan_module_details_t);
+  swiftscan_string_set_t *
+  (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t);
   bool
   (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t);
 

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -234,6 +234,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
     for dependencyModule in swiftDependencyArtifacts {
       inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path,
                                      type: .swiftModule))
+
+      for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] {
+        commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"])
+        commandLine.appendPath(VirtualPath.lookup(headerDep.path))
+        inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch))
+      }
     }
 
     // Clang module dependencies are specified on the command line explicitly
@@ -301,6 +307,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
         swiftDependencyArtifacts.append(
           SwiftModuleArtifactInfo(name: dependencyId.moduleName,
                                   modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
+                                  headerDependencies: prebuiltModuleDetails.headerDependencyPaths,
                                   isFramework: isFramework))
       case .swiftPlaceholder:
         fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -146,16 +146,21 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable {
   /// The path to the .swiftSourceInfo file.
   public var moduleSourceInfoPath: TextualVirtualPath?
 
+  /// The paths to the binary module's header dependencies
+  public var headerDependencyPaths: [TextualVirtualPath]?
+
   /// A flag to indicate whether or not this module is a framework.
   public var isFramework: Bool?
 
   public init(compiledModulePath: TextualVirtualPath,
               moduleDocPath: TextualVirtualPath? = nil,
               moduleSourceInfoPath: TextualVirtualPath? = nil,
+              headerDependencies: [TextualVirtualPath]? = nil,
               isFramework: Bool) throws {
     self.compiledModulePath = compiledModulePath
     self.moduleDocPath = moduleDocPath
     self.moduleSourceInfoPath = moduleSourceInfoPath
+    self.headerDependencyPaths = headerDependencies
     self.isFramework = isFramework
   }
 }

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -132,16 +132,23 @@ public class InterModuleDependencyOracle {
 
   @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool {
     guard let swiftScan = swiftScanLibInstance else {
-      fatalError("Attempting to reset scanner cache with no scanner instance.")
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.supportsScannerDiagnostics
+  }
+
+  @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
     }
-    return swiftScan.supportsScannerDiagnostics()
+    return swiftScan.supportsBinaryModuleHeaderDependencies
   }
 
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")
     }
-    guard swiftScan.supportsScannerDiagnostics() else {
+    guard swiftScan.supportsScannerDiagnostics else {
       return nil
     }
     let diags = try swiftScan.queryScannerDiagnostics()

Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift

@@ -24,15 +24,19 @@
   public let docPath: TextualVirtualPath?
   /// The path for the module's .swiftsourceinfo file
   public let sourceInfoPath: TextualVirtualPath?
+  /// Header dependencies of this module
+  public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]?
   /// A flag to indicate whether this module is a framework
   public let isFramework: Bool
 
   init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil,
-       sourceInfoPath: TextualVirtualPath? = nil, isFramework: Bool = false) {
+       sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil,
+       isFramework: Bool = false) {
     self.moduleName = name
     self.modulePath = modulePath
     self.docPath = docPath
     self.sourceInfoPath = sourceInfoPath
+    self.prebuiltHeaderDependencyPaths = headerDependencies
     self.isFramework = isFramework
   }
 }

Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

@@ -195,7 +195,7 @@ private extension SwiftScan {
 
     // Decode all dependencies of this module
     let swiftOverlayDependencies: [ModuleDependencyId]?
-    if supportsSeparateSwiftOverlayDependencies(),
+    if supportsSeparateSwiftOverlayDependencies,
        let encodedOverlayDepsRef = api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies(moduleDetailsRef) {
       let encodedOverlayDependencies = try toSwiftStringArray(encodedOverlayDepsRef.pointee)
       swiftOverlayDependencies =
@@ -228,6 +228,14 @@ private extension SwiftScan {
       try getOptionalPathDetail(from: moduleDetailsRef,
                                 using: api.swiftscan_swift_binary_detail_get_module_source_info_path)
 
+    let headerDependencies: [TextualVirtualPath]?
+    if supportsBinaryModuleHeaderDependencies {
+      headerDependencies = try getOptionalPathArrayDetail(from: moduleDetailsRef,
+                                                          using: api.swiftscan_swift_binary_detail_get_header_dependencies)
+    } else {
+      headerDependencies = nil
+    }
+
     let isFramework: Bool
     if hasBinarySwiftModuleIsFramework {
       isFramework = api.swiftscan_swift_binary_detail_get_is_framework(moduleDetailsRef)
@@ -238,6 +246,7 @@ private extension SwiftScan {
     return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath,
                                                   moduleDocPath: moduleDocPath,
                                                   moduleSourceInfoPath: moduleSourceInfoPath,
+                                                  headerDependencies: headerDependencies,
                                                   isFramework: isFramework)
   }
 

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -250,34 +250,39 @@ internal extension swiftscan_diagnostic_severity_t {
     api.swiftscan_clang_detail_get_captured_pcm_args != nil
   }
 
-  func serializeScannerCache(to path: AbsolutePath) {
-    api.swiftscan_scanner_cache_serialize(scanner,
-                                          path.description.cString(using: String.Encoding.utf8))
+  @_spi(Testing) public var supportsBinaryModuleHeaderDependencies : Bool {
+    return api.swiftscan_swift_binary_detail_get_header_dependencies != nil
   }
 
-  func loadScannerCache(from path: AbsolutePath) -> Bool {
-    return api.swiftscan_scanner_cache_load(scanner,
-                                            path.description.cString(using: String.Encoding.utf8))
+  @_spi(Testing) public var supportsStringDispose : Bool {
+    return api.swiftscan_string_dispose != nil
   }
 
-  func resetScannerCache() {
-    api.swiftscan_scanner_cache_reset(scanner)
-  }
 
-  @_spi(Testing) public func supportsSeparateSwiftOverlayDependencies() -> Bool {
+  @_spi(Testing) public var supportsSeparateSwiftOverlayDependencies : Bool {
     return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil
   }
-  
-  @_spi(Testing) public func supportsScannerDiagnostics() -> Bool {
+
+  @_spi(Testing) public var supportsScannerDiagnostics : Bool {
     return api.swiftscan_scanner_diagnostics_query != nil &&
            api.swiftscan_scanner_diagnostics_reset != nil &&
            api.swiftscan_diagnostic_get_message != nil &&
            api.swiftscan_diagnostic_get_severity != nil &&
            api.swiftscan_diagnostics_set_dispose != nil
   }
 
-  @_spi(Testing) public func supportsStringDispose() -> Bool {
-    return api.swiftscan_string_dispose != nil
+  func serializeScannerCache(to path: AbsolutePath) {
+    api.swiftscan_scanner_cache_serialize(scanner,
+                                          path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func loadScannerCache(from path: AbsolutePath) -> Bool {
+    return api.swiftscan_scanner_cache_load(scanner,
+                                            path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func resetScannerCache() {
+    api.swiftscan_scanner_cache_reset(scanner)
   }
 
   @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
@@ -427,6 +432,10 @@ private extension swiftscan_functions_t {
     self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies =
       try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies")
 
+    // Header dependencies of binary modules
+    self.swiftscan_swift_binary_detail_get_header_dependencies =
+      try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -904,7 +904,83 @@ final class ExplicitModuleBuildTests: XCTestCase {
       XCTAssertTrue(FileManager.default.fileExists(atPath: moduleFooPath))
     }
   }
-  
+
+  func testExplicitModuleBuildEndToEndWithBinaryHeaderDeps() throws {
+    try withTemporaryDirectory { path in
+      try localFileSystem.changeCurrentWorkingDirectory(to: path)
+      let moduleCachePath = path.appending(component: "ModuleCache")
+      try localFileSystem.createDirectory(moduleCachePath)
+      let PCHPath = path.appending(component: "PCH")
+      try localFileSystem.createDirectory(PCHPath)
+      let FooInstallPath = path.appending(component: "Foo")
+      try localFileSystem.createDirectory(FooInstallPath)
+      let foo = path.appending(component: "foo.swift")
+      try localFileSystem.writeFileContents(foo) {
+        $0 <<< "extension Profiler {"
+        $0 <<< "    public static let count: Int = 42"
+        $0 <<< "}"
+      }
+      let fooHeader = path.appending(component: "foo.h")
+      try localFileSystem.writeFileContents(fooHeader) {
+        $0 <<< "struct Profiler { void* ptr; };"
+      }
+      let main = path.appending(component: "main.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import Foo"
+      }
+      let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? []
+
+      var fooBuildDriver = try Driver(args: ["swiftc",
+                                             "-explicit-module-build",
+                                             "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                             "-working-directory", path.nativePathString(escaped: true),
+                                             foo.nativePathString(escaped: true),
+                                             "-emit-module", "-wmo", "-module-name", "Foo",
+                                             "-emit-module-path", FooInstallPath.nativePathString(escaped: true),
+                                             "-import-objc-header", fooHeader.nativePathString(escaped: true),
+                                             "-pch-output-dir", PCHPath.nativePathString(escaped: true),
+                                             FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)]
+                                      + sdkArgumentsForTesting,
+                                      env: ProcessEnv.vars)
+
+      // Ensure this tooling supports this functionality
+      let dependencyOracle = InterModuleDependencyOracle()
+      let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib())
+      guard try dependencyOracle
+              .verifyOrCreateScannerInstance(fileSystem: localFileSystem,
+                                             swiftScanLibPath: scanLibPath) else {
+        XCTFail("Dependency scanner library not found")
+        return
+      }
+      guard try dependencyOracle.supportsBinaryModuleHeaderDependencies() else {
+        throw XCTSkip("libSwiftScan does not support binary module header dependencies.")
+      }
+
+      let fooJobs = try fooBuildDriver.planBuild()
+      try fooBuildDriver.run(jobs: fooJobs)
+      XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
+
+      var driver = try Driver(args: ["swiftc",
+                                     "-I", FooInstallPath.nativePathString(escaped: true),
+                                     "-explicit-module-build", "-emit-module", "-emit-module-path",
+                                     path.appending(component: "testEMBETEWBHD.swiftmodule").nativePathString(escaped: true),
+                                     "-module-cache-path", moduleCachePath.nativePathString(escaped: true),
+                                     "-working-directory", path.nativePathString(escaped: true),
+                                     main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
+                              env: ProcessEnv.vars)
+      let jobs = try driver.planBuild()
+      let compileJob = try XCTUnwrap(jobs.first(where: { $0.description == "Compiling main main.swift" }))
+
+      // Ensure the header dependency of Foo shows up on client compile commands
+      XCTAssertTrue(compileJob.commandLine.contains(subsequence: [.flag("-Xcc"),
+                                                                  .flag("-include-pch"),
+                                                                  .flag("-Xcc"),
+                                                                  .path(.absolute(PCHPath.appending(component: "foo.pch")))]))
+      try driver.run(jobs: jobs)
+      XCTAssertFalse(driver.diagnosticEngine.hasErrors)
+    }
+  }
+
   func testExplicitModuleBuildEndToEnd() throws {
     try withTemporaryDirectory { path in
       try localFileSystem.changeCurrentWorkingDirectory(to: path)