[Dependency Scanning] Record header dependencies of binary Swift module dependencies

include/swift-c/DependencyScan/DependencyScan.h

@@ -180,6 +180,10 @@ SWIFTSCAN_PUBLIC swiftscan_string_ref_t
 swiftscan_swift_binary_detail_get_module_source_info_path(
     swiftscan_module_details_t details);
 
+SWIFTSCAN_PUBLIC swiftscan_string_set_t *
+swiftscan_swift_binary_detail_get_header_dependencies(
+    swiftscan_module_details_t details);
+
 SWIFTSCAN_PUBLIC bool
 swiftscan_swift_binary_detail_get_is_framework(
     swiftscan_module_details_t details);

include/swift/AST/ModuleDependencies.h

@@ -106,10 +106,15 @@ class ModuleDependencyInfoStorageBase {
   const ModuleDependencyKind dependencyKind;
 
   ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
-                                  StringRef moduleCacheKey = "",
-                                  bool resolved = false)
+                                  StringRef moduleCacheKey = "")
       : dependencyKind(dependencyKind), moduleCacheKey(moduleCacheKey.str()),
-        resolved(resolved) {}
+        resolved(false) { }
+
+  ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
+                                  const std::vector<std::string> &moduleImports,
+                                  StringRef moduleCacheKey = "")
+      : dependencyKind(dependencyKind), moduleImports(moduleImports),
+        moduleCacheKey(moduleCacheKey.str()), resolved(false)  {}
 
   virtual ModuleDependencyInfoStorageBase *clone() const = 0;
 
@@ -282,12 +287,15 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas
   SwiftBinaryModuleDependencyStorage(const std::string &compiledModulePath,
                                      const std::string &moduleDocPath,
                                      const std::string &sourceInfoPath,
+                                     const std::vector<std::string> &moduleImports,
+                                     const std::vector<std::string> &headerImports,
                                      const bool isFramework,
                                      const std::string &moduleCacheKey)
       : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary,
-                                        moduleCacheKey),
+                                        moduleImports, moduleCacheKey),
         compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
-        sourceInfoPath(sourceInfoPath), isFramework(isFramework) {}
+        sourceInfoPath(sourceInfoPath), preCompiledBridgingHeaderPaths(headerImports),
+        isFramework(isFramework) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
     return new SwiftBinaryModuleDependencyStorage(*this);
@@ -302,6 +310,9 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas
   /// The path to the .swiftSourceInfo file.
   const std::string sourceInfoPath;
 
+  /// The paths of all the .pch dependencies of this module.
+  const std::vector<std::string> preCompiledBridgingHeaderPaths;
+
   /// A flag that indicates this dependency is a framework
   const bool isFramework;
 
@@ -447,15 +458,17 @@ class ModuleDependencyInfo {
   }
 
   /// Describe the module dependencies for a serialized or parsed Swift module.
-  static ModuleDependencyInfo
-  forSwiftBinaryModule(const std::string &compiledModulePath,
-                       const std::string &moduleDocPath,
-                       const std::string &sourceInfoPath, bool isFramework,
-                       const std::string &moduleCacheKey) {
+  static ModuleDependencyInfo forSwiftBinaryModule(
+      const std::string &compiledModulePath,
+      const std::string &moduleDocPath,
+      const std::string &sourceInfoPath,
+      const std::vector<std::string> &moduleImports,
+      const std::vector<std::string> &headerImports,
+      bool isFramework, const std::string &moduleCacheKey) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftBinaryModuleDependencyStorage>(
-            compiledModulePath, moduleDocPath, sourceInfoPath, isFramework,
-            moduleCacheKey));
+          compiledModulePath, moduleDocPath, sourceInfoPath,
+          moduleImports, headerImports, isFramework, moduleCacheKey));
   }
 
   /// Describe the main Swift module.

include/swift/DependencyScan/DependencyScanImpl.h

@@ -121,6 +121,10 @@ typedef struct {
   /// The path to the .swiftSourceInfo file.
   swiftscan_string_ref_t module_source_info_path;
 
+  /// (Clang) header dependencies of this binary module.
+  /// Typically pre-compiled bridging header.
+  swiftscan_string_set_t *header_dependencies;
+
   /// A flag to indicate whether or not this module is a framework.
   bool is_framework;
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -165,6 +165,7 @@ using SwiftBinaryModuleDetailsLayout =
                    FileIDField,                      // compiledModulePath
                    FileIDField,                      // moduleDocPath
                    FileIDField,                      // moduleSourceInfoPath
+                   ImportArrayIDField,               // headerImports
                    IsFrameworkField,                 // isFramework
                    IdentifierIDField                 // moduleCacheKey
                    >;

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -241,12 +241,14 @@ struct ExplicitSwiftModuleInputInfo {
   ExplicitSwiftModuleInputInfo(std::string modulePath,
                                llvm::Optional<std::string> moduleDocPath,
                                llvm::Optional<std::string> moduleSourceInfoPath,
+                               llvm::Optional<std::vector<std::string>> headerDependencyPaths,
                                bool isFramework = false,
                                bool isSystem = false,
                                llvm::Optional<std::string> moduleCacheKey = None)
     : modulePath(modulePath),
       moduleDocPath(moduleDocPath),
       moduleSourceInfoPath(moduleSourceInfoPath),
+      headerDependencyPaths(headerDependencyPaths),
       isFramework(isFramework),
       isSystem(isSystem),
       moduleCacheKey(moduleCacheKey) {}
@@ -256,6 +258,8 @@ struct ExplicitSwiftModuleInputInfo {
   llvm::Optional<std::string> moduleDocPath;
   // Path of the .swiftsourceinfo file.
   llvm::Optional<std::string> moduleSourceInfoPath;
+  // Paths of the precompiled header dependencies of this module.
+  llvm::Optional<std::vector<std::string>> headerDependencyPaths;
   // A flag that indicates whether this module is a framework
   bool isFramework = false;
   // A flag that indicates whether this module is a system module
@@ -369,34 +373,39 @@ class ExplicitModuleMapParser {
     llvm::Optional<std::string> swiftModulePath, swiftModuleDocPath,
                                 swiftModuleSourceInfoPath, swiftModuleCacheKey,
                                 clangModuleCacheKey;
+    llvm::Optional<std::vector<std::string>> headerDependencyPaths;
     std::string clangModuleMapPath = "", clangModulePath = "";
     bool isFramework = false, isSystem = false;
     for (auto &entry : *mapNode) {
       auto key = getScalaNodeText(entry.getKey());
-      auto val = getScalaNodeText(entry.getValue());
-      if (key == "moduleName") {
-        moduleName = val;
-      } else if (key == "modulePath") {
-        swiftModulePath = val.str();
-      } else if (key == "docPath") {
-        swiftModuleDocPath = val.str();
-      } else if (key == "sourceInfoPath") {
-        swiftModuleSourceInfoPath = val.str();
-      } else if (key == "isFramework") {
-        isFramework = parseBoolValue(val);
-      } else if (key == "isSystem") {
-        isSystem = parseBoolValue(val);
-      } else if (key == "clangModuleMapPath") {
-        clangModuleMapPath = val.str();
-      } else if (key == "clangModulePath") {
-        clangModulePath = val.str();
-      } else if (key == "moduleCacheKey") {
-        swiftModuleCacheKey = val.str();
-      } else if (key == "clangModuleCacheKey") {
-        clangModuleCacheKey = val.str();
-      } else {
-        // Being forgiving for future fields.
+      if (key == "prebuiltHeaderDependencyPaths") {
         continue;
+      } else {
+        auto val = getScalaNodeText(entry.getValue());
+        if (key == "moduleName") {
+          moduleName = val;
+        } else if (key == "modulePath") {
+          swiftModulePath = val.str();
+        } else if (key == "docPath") {
+          swiftModuleDocPath = val.str();
+        } else if (key == "sourceInfoPath") {
+          swiftModuleSourceInfoPath = val.str();
+        } else if (key == "isFramework") {
+          isFramework = parseBoolValue(val);
+        } else if (key == "isSystem") {
+          isSystem = parseBoolValue(val);
+        } else if (key == "clangModuleMapPath") {
+          clangModuleMapPath = val.str();
+        } else if (key == "clangModulePath") {
+          clangModulePath = val.str();
+        } else if (key == "moduleCacheKey") {
+          swiftModuleCacheKey = val.str();
+        } else if (key == "clangModuleCacheKey") {
+          clangModuleCacheKey = val.str();
+        } else {
+          // Being forgiving for future fields.
+          continue;
+        }
       }
     }
     if (moduleName.empty())
@@ -409,6 +418,7 @@ class ExplicitModuleMapParser {
       ExplicitSwiftModuleInputInfo entry(swiftModulePath.value(),
                                          swiftModuleDocPath,
                                          swiftModuleSourceInfoPath,
+                                         headerDependencyPaths,
                                          isFramework,
                                          isSystem,
                                          swiftModuleCacheKey);

include/swift/Serialization/SerializedModuleLoader.h

@@ -150,15 +150,20 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// Scan the given serialized module file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo> scanModuleFile(Twine modulePath, bool isFramework);
 
-  static llvm::ErrorOr<llvm::StringSet<>>
-  getModuleImportsOfModule(Twine modulePath,
-                           ModuleLoadingBehavior transitiveBehavior,
-                           bool isFramework,
-                           bool isRequiredOSSAModules,
-                           StringRef SDKName,
-                           StringRef packageName,
-                           llvm::vfs::FileSystem *fileSystem,
-                           PathObfuscator &recoverer);
+  struct BinaryModuleImports {
+    llvm::StringSet<> moduleImports;
+    llvm::StringSet<> headerImports;
+  };
+
+  static llvm::ErrorOr<BinaryModuleImports>
+  getImportsOfModule(Twine modulePath,
+                     ModuleLoadingBehavior transitiveBehavior,
+                     bool isFramework,
+                     bool isRequiredOSSAModules,
+                     StringRef SDKName,
+                     StringRef packageName,
+                     llvm::vfs::FileSystem *fileSystem,
+                     PathObfuscator &recoverer);
 
   /// Load the module file into a buffer and also collect its module name.
   static std::unique_ptr<llvm::MemoryBuffer>

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -462,10 +462,11 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
             "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
       unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
-          isFramework, moduleCacheKeyID;
+               headerImportsArrayID, isFramework, moduleCacheKeyID;
       SwiftBinaryModuleDetailsLayout::readRecord(
           Scratch, compiledModulePathID, moduleDocPathID,
-          moduleSourceInfoPathID, isFramework, moduleCacheKeyID);
+          moduleSourceInfoPathID, headerImportsArrayID, isFramework,
+          moduleCacheKeyID);
 
       auto compiledModulePath = getIdentifier(compiledModulePathID);
       if (!compiledModulePath)
@@ -477,16 +478,18 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!moduleSourceInfoPath)
         llvm::report_fatal_error("Bad module source info path");
       auto moduleCacheKey = getIdentifier(moduleCacheKeyID);
-      if (!moduleCacheKeyID)
+      if (!moduleCacheKey)
         llvm::report_fatal_error("Bad moduleCacheKey");
 
+      auto headerImports = getStringArray(headerImportsArrayID);
+      if (!headerImports)
+        llvm::report_fatal_error("Bad binary direct dependencies: no header imports");
+
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
-          *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
-          isFramework, *moduleCacheKey);
-      // Add dependencies of this module
-      for (const auto &moduleName : *currentModuleImports)
-        moduleDep.addModuleImport(moduleName);
+           *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
+           *currentModuleImports, *headerImports, isFramework,
+           *moduleCacheKey);
 
       cache.recordDependency(currentModuleName, std::move(moduleDep),
                              getContextHash());
@@ -704,6 +707,7 @@ bool swift::dependencies::module_dependency_cache_serialization::
 enum ModuleIdentifierArrayKind : uint8_t {
   Empty = 0,
   DependencyImports,
+  DependencyHeaders,
   QualifiedModuleDependencyIDs,
   CompiledModuleCandidates,
   BuildCommandLine,
@@ -975,6 +979,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
         getIdentifier(swiftBinDeps->compiledModulePath),
         getIdentifier(swiftBinDeps->moduleDocPath),
         getIdentifier(swiftBinDeps->sourceInfoPath),
+        getArrayID(moduleID, ModuleIdentifierArrayKind::DependencyHeaders),
         swiftBinDeps->isFramework,
         getIdentifier(swiftBinDeps->moduleCacheKey));
 
@@ -1152,6 +1157,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->moduleDocPath);
         addIdentifier(swiftBinDeps->sourceInfoPath);
         addIdentifier(swiftBinDeps->moduleCacheKey);
+        addStringArray(moduleID, ModuleIdentifierArrayKind::DependencyHeaders,
+                       swiftBinDeps->preCompiledBridgingHeaderPaths);
         break;
       }
       case swift::ModuleDependencyKind::SwiftPlaceholder: {

lib/DependencyScan/ScanDependencies.cpp

@@ -254,6 +254,10 @@ computeTransitiveClosureOfExplicitDependencies(
       llvm::set_union(modReachableSet, succReachableSet);
     }
   }
+  // For ease of use down-the-line, remove the node's self from its set of reachable nodes
+  for (const auto &modID : topologicallySortedModuleList)
+    result[modID].erase(modID);
+
   return result;
 }
 
@@ -330,6 +334,12 @@ static llvm::Error resolveExplicitModuleInputs(
                        : binaryDepDetails->moduleCacheKey;
       commandLine.push_back("-swift-module-file=" + depModuleID.first + "=" +
                             path);
+      for (const auto &headerDep : binaryDepDetails->preCompiledBridgingHeaderPaths) {
+        commandLine.push_back("-Xcc");
+        commandLine.push_back("-include-pch");
+        commandLine.push_back("-Xcc");
+        commandLine.push_back(headerDep);
+      }
     } break;
     case swift::ModuleDependencyKind::SwiftPlaceholder: {
       auto placeholderDetails = depInfo->getAsPlaceholderDependencyModule();
@@ -1163,6 +1173,13 @@ static void writeJSON(llvm::raw_ostream &out,
                              swiftBinaryDeps->module_cache_key, 5,
                              /*trailingComma=*/true);
       }
+
+      // Module Header Dependencies
+      if (swiftBinaryDeps->header_dependencies->count != 0)
+        writeJSONSingleField(out, "headerDependencies",
+                             swiftBinaryDeps->header_dependencies, 5,
+                             /*trailingComma=*/true);
+
       writeJSONSingleField(out, "isFramework", swiftBinaryDeps->is_framework,
                            5, /*trailingComma=*/false);
     } else {
@@ -1399,6 +1416,7 @@ generateFullDependencyGraph(CompilerInstance &instance,
             create_clone(swiftBinaryDeps->compiledModulePath.c_str()),
             create_clone(swiftBinaryDeps->moduleDocPath.c_str()),
             create_clone(swiftBinaryDeps->sourceInfoPath.c_str()),
+            create_set(swiftBinaryDeps->preCompiledBridgingHeaderPaths),
             swiftBinaryDeps->isFramework,
             create_clone(swiftBinaryDeps->moduleCacheKey.c_str())};
       } else {

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -2004,7 +2004,7 @@ struct ExplicitSwiftModuleLoader::Implementation {
       const std::vector<std::pair<std::string, std::string>>
           &commandLineExplicitInputs) {
     for (const auto &moduleInput : commandLineExplicitInputs) {
-      ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {});
+      ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {}, {});
       ExplicitModuleMap.try_emplace(moduleInput.first, std::move(entry));
     }
   }
@@ -2270,7 +2270,7 @@ struct ExplicitCASModuleLoader::Implementation {
       const std::vector<std::pair<std::string, std::string>>
           &commandLineExplicitInputs) {
     for (const auto &moduleInput : commandLineExplicitInputs) {
-      ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {});
+      ExplicitSwiftModuleInputInfo entry(moduleInput.second, {}, {}, {});
       ExplicitModuleMap.try_emplace(moduleInput.first, std::move(entry));
     }
   }