Merge pull request #73064 from cachemeifyoucan/eng/PR-swift-caching-6.0-update

[6.0][Caching] Add swift caching changes to swift 6.0

Author: cachemeifyoucan

include/swift/AST/DiagnosticsClangImporter.def

@@ -42,6 +42,8 @@ WARNING(could_not_rewrite_bridging_header,none,
 ERROR(bridging_header_pch_error,Fatal,
    "failed to emit precompiled header '%0' for bridging header '%1'",
    (StringRef, StringRef))
+ERROR(err_rewrite_bridging_header,none,
+  "failed to serialize bridging header: '%0'", (StringRef))
 
 ERROR(emit_pcm_error,Fatal,
    "failed to emit precompiled module '%0' for module map '%1'",

include/swift/AST/DiagnosticsSema.def

@@ -1193,6 +1193,9 @@ REMARK(module_api_import_aliases,none,
        "%select{, which reexports definition from %2|}3",
        (const Decl *, ModuleDecl *, ModuleDecl *, bool))
 
+REMARK(skip_module_invalid,none,"skip invalid swiftmodule: %0", (StringRef))
+REMARK(skip_module_testable,none,"skip swiftmodule built with '-enable-testing': %0", (StringRef))
+
 REMARK(macro_loaded,none,
        "loaded macro implementation module %0 from "
        "%select{shared library '%2'|executable '%2'|"

include/swift/AST/ModuleDependencies.h

@@ -1097,6 +1097,12 @@ class ModuleDependenciesCache {
   std::optional<const ModuleDependencyInfo *>
   findDependency(StringRef moduleName) const;
 
+  /// Look for known existing dependencies.
+  ///
+  /// \returns the cached result.
+  const ModuleDependencyInfo &
+  findKnownDependency(const ModuleDependencyID &moduleID) const;
+
   /// Record dependencies for the given module.
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);

include/swift/AST/SearchPathOptions.h

@@ -38,6 +38,15 @@ enum class ModuleSearchPathKind {
   RuntimeLibrary,
 };
 
+/// Specifies how to load modules when both a module interface and serialized
+/// AST are present, or whether to disallow one format or the other altogether.
+enum class ModuleLoadingMode {
+  PreferInterface,
+  PreferSerialized,
+  OnlyInterface,
+  OnlySerialized
+};
+
 /// A single module search path that can come from different sources, e.g.
 /// framework search paths, import search path etc.
 class ModuleSearchPath : public llvm::RefCountedBase<ModuleSearchPath> {
@@ -499,6 +508,12 @@ class SearchPathOptions {
   /// original form.
   PathObfuscator DeserializedPathRecoverer;
 
+  /// Specify the module loading behavior of the compilation.
+  ModuleLoadingMode ModuleLoadMode = ModuleLoadingMode::PreferSerialized;
+
+  /// Legacy scanner search behavior.
+  bool NoScannerModuleValidation = false;
+
   /// Return all module search paths that (non-recursively) contain a file whose
   /// name is in \p Filenames.
   SmallVector<const ModuleSearchPath *, 4>
@@ -546,7 +561,9 @@ class SearchPathOptions {
                         RuntimeResourcePath,
                         hash_combine_range(RuntimeLibraryImportPaths.begin(),
                                            RuntimeLibraryImportPaths.end()),
-                        DisableModulesValidateSystemDependencies);
+                        DisableModulesValidateSystemDependencies,
+                        NoScannerModuleValidation,
+                        ModuleLoadMode);
   }
 
   /// Return a hash code of any components from these options that should

include/swift/ClangImporter/ClangImporter.h

@@ -402,8 +402,10 @@ class ClangImporter final : public ClangModuleLoader {
   getWrapperForModule(const clang::Module *mod,
                       bool returnOverlayIfPossible = false) const override;
 
-  std::string getBridgingHeaderContents(StringRef headerPath, off_t &fileSize,
-                                        time_t &fileModTime);
+  std::string
+  getBridgingHeaderContents(StringRef headerPath, off_t &fileSize,
+                            time_t &fileModTime,
+                            StringRef pchIncludeTree);
 
   /// Makes a temporary replica of the ClangImporter's CompilerInstance, reads
   /// an Objective-C header file into the replica and emits a PCH file of its

include/swift/Option/FrontendOptions.td

@@ -1337,4 +1337,10 @@ def disable_strict_concurrency_region_based_isolation : Flag<["-"],
   HelpText<"Disable region based isolation when running with strict concurrency enabled. Only enabled with asserts">,
   Flags<[HelpHidden]>;
 
+def no_scanner_module_validation: Flag<["-"], "no-scanner-module-validation">,
+    HelpText<"Do not validate binary modules in scanner and delegate the validation to swift-frontend">;
+def module_load_mode: Separate<["-"], "module-load-mode">,
+    MetaVarName<"only-interface|prefer-interface|prefer-serialized|only-serialized">,
+    HelpText<"Module loading mode">;
+
 } // end let Flags = [FrontendOption, NoDriverOption, HelpHidden]

include/swift/Serialization/ScanningLoaders.h

@@ -34,7 +34,8 @@ class SwiftModuleScanner : public SerializedModuleLoaderBase {
 
   /// Scan the given interface file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo>
-  scanInterfaceFile(Twine moduleInterfacePath, bool isFramework);
+  scanInterfaceFile(Twine moduleInterfacePath, bool isFramework,
+                    bool isTestableImport);
 
   InterfaceSubContextDelegate &astDelegate;
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -16,27 +16,20 @@
 #include "swift/AST/FileUnit.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/ModuleLoader.h"
+#include "swift/AST/SearchPathOptions.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrefixMapper.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace swift {
 class ModuleFile;
 class PathObfuscator;
+class ModuleFileSharedCore;
 enum class ModuleLoadingBehavior;
 namespace file_types {
   enum ID : uint8_t;
 }
 
-/// Specifies how to load modules when both a module interface and serialized
-/// AST are present, or whether to disallow one format or the other altogether.
-enum class ModuleLoadingMode {
-  PreferInterface,
-  PreferSerialized,
-  OnlyInterface,
-  OnlySerialized
-};
-
 /// How a dependency should be loaded.
 ///
 /// \sa getTransitiveLoadingBehavior
@@ -170,22 +163,18 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   }
 
   /// Scan the given serialized module file to determine dependencies.
-  llvm::ErrorOr<ModuleDependencyInfo> scanModuleFile(Twine modulePath, bool isFramework);
+  llvm::ErrorOr<ModuleDependencyInfo>
+  scanModuleFile(Twine modulePath, bool isFramework, bool isTestableImport);
 
   struct BinaryModuleImports {
     llvm::StringSet<> moduleImports;
     std::string headerImport;
   };
 
-  static llvm::ErrorOr<BinaryModuleImports>
-  getImportsOfModule(Twine modulePath,
+  static BinaryModuleImports
+  getImportsOfModule(const ModuleFileSharedCore &loadedModule,
                      ModuleLoadingBehavior transitiveBehavior,
-                     bool isFramework,
-                     bool isRequiredOSSAModules,
-                     StringRef SDKName,
-                     StringRef packageName,
-                     llvm::vfs::FileSystem *fileSystem,
-                     PathObfuscator &recoverer);
+                     StringRef packageName, bool isTestableImport);
 
   /// Load the module file into a buffer and also collect its module name.
   static std::unique_ptr<llvm::MemoryBuffer>

lib/AST/ModuleDependencies.cpp

@@ -21,6 +21,7 @@
 #include "swift/Frontend/Frontend.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrefixMapper.h"
@@ -480,6 +481,58 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
   // Add VFSOverlay file.
   for (auto &Overlay: Opts.VFSOverlayFiles)
     FS->status(Overlay);
+
+  // Add plugin dylibs from the toolchain only by look through the plugin search
+  // directory.
+  auto recordFiles = [&](StringRef Path) {
+    std::error_code EC;
+    for (auto I = FS->dir_begin(Path, EC);
+         !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
+      if (I->type() != llvm::sys::fs::file_type::regular_file)
+        continue;
+#if defined(_WIN32)
+      constexpr StringRef libPrefix{};
+      constexpr StringRef libSuffix = ".dll";
+#else
+      constexpr StringRef libPrefix = "lib";
+      constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
+#endif
+      StringRef filename = llvm::sys::path::filename(I->path());
+      if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
+        FS->status(I->path());
+    }
+  };
+  for (auto &entry : Opts.PluginSearchOpts) {
+    switch (entry.getKind()) {
+
+    // '-load-plugin-library <library path>'.
+    case PluginSearchOption::Kind::LoadPluginLibrary: {
+      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
+      FS->status(val.LibraryPath);
+      break;
+    }
+
+    // '-load-plugin-executable <executable path>#<module name>, ...'.
+    case PluginSearchOption::Kind::LoadPluginExecutable: {
+      // We don't have executable plugin in toolchain.
+      break;
+    }
+
+    // '-plugin-path <library search path>'.
+    case PluginSearchOption::Kind::PluginPath: {
+      auto &val = entry.get<PluginSearchOption::PluginPath>();
+      recordFiles(val.SearchPath);
+      break;
+    }
+
+    // '-external-plugin-path <library search path>#<server path>'.
+    case PluginSearchOption::Kind::ExternalPluginPath: {
+      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
+      recordFiles(val.SearchPath);
+      break;
+    }
+    }
+  }
 }
 
 void SwiftDependencyTracker::startTracking() {
@@ -739,6 +792,13 @@ ModuleDependenciesCache::findDependency(StringRef moduleName) const {
   return std::nullopt;
 }
 
+const ModuleDependencyInfo &ModuleDependenciesCache::findKnownDependency(
+    const ModuleDependencyID &moduleID) const {
+  auto dep = findDependency(moduleID);
+  assert(dep && "dependency unknown");
+  return **dep;
+}
+
 bool ModuleDependenciesCache::hasDependency(const ModuleDependencyID &moduleID) const {
   return hasDependency(moduleID.ModuleName, moduleID.Kind);
 }

lib/AST/PluginLoader.cpp

@@ -17,6 +17,7 @@
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/Lexer.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 using namespace swift;
 
@@ -59,6 +60,15 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {
   return "";
 }
 
+static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+getPluginLoadingFS(ASTContext &Ctx) {
+  // If there is a clang include tree FS, using real file system to load plugin
+  // as the FS in SourceMgr doesn't support directory iterator.
+  if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
+    return llvm::vfs::getRealFileSystem();
+  return Ctx.SourceMgr.getFileSystem();
+}
+
 llvm::DenseMap<Identifier, PluginLoader::PluginEntry> &
 PluginLoader::getPluginMap() {
   if (PluginMap.has_value()) {
@@ -86,7 +96,7 @@ PluginLoader::getPluginMap() {
     (void)result;
   };
 
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   std::error_code ec;
 
   for (auto &entry : Ctx.SearchPathOpts.PluginSearchOpts) {
@@ -162,7 +172,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) {
 
 llvm::Expected<LoadedLibraryPlugin *>
 PluginLoader::loadLibraryPlugin(StringRef path) {
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   SmallString<128> resolvedPath;
   if (auto err = fs->getRealPath(path, resolvedPath)) {
     return llvm::createStringError(err, err.message());
@@ -186,7 +196,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) {
 
 llvm::Expected<LoadedExecutablePlugin *>
 PluginLoader::loadExecutablePlugin(StringRef path) {
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   SmallString<128> resolvedPath;
   if (auto err = fs->getRealPath(path, resolvedPath)) {
     return llvm::createStringError(err, err.message());