Merge pull request #32170 from nkcsgexi/explicit-module-loader

Front-end: add a new module loader that loads explicitly built Swift modules

Author: nkcsgexi

include/swift/AST/DiagnosticsFrontend.def

@@ -326,6 +326,8 @@ ERROR(error_extracting_version_from_module_interface,none,
 ERROR(unsupported_version_of_module_interface,none,
       "unsupported version of module interface '%0': '%1'",
       (StringRef, llvm::VersionTuple))
+ERROR(error_opening_explicit_module_file,none,
+      "failed to open explicit Swift module: %0", (StringRef))
 ERROR(error_extracting_flags_from_module_interface,none,
       "error extracting flags from module interface", ())
 REMARK(rebuilding_module_from_interface,none,

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -127,6 +127,38 @@ class LangOptions;
 class SearchPathOptions;
 class CompilerInvocation;
 
+/// A ModuleLoader that loads explicitly built Swift modules specified via
+/// -swift-module-file
+class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
+  explicit ExplicitSwiftModuleLoader(ASTContext &ctx, DependencyTracker *tracker,
+                                     ModuleLoadingMode loadMode,
+                                     bool IgnoreSwiftSourceInfoFile);
+  std::error_code findModuleFilesInDirectory(
+    AccessPathElem ModuleID,
+    const SerializedModuleBaseName &BaseName,
+    SmallVectorImpl<char> *ModuleInterfacePath,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) override;
+
+  bool isCached(StringRef DepPath) override { return false; };
+
+  struct Implementation;
+  Implementation &Impl;
+public:
+  static std::unique_ptr<ExplicitSwiftModuleLoader>
+  create(ASTContext &ctx,
+         DependencyTracker *tracker, ModuleLoadingMode loadMode,
+         ArrayRef<std::string> ExplicitModulePaths,
+         bool IgnoreSwiftSourceInfoFile);
+
+  /// Append visible module names to \p names. Note that names are possibly
+  /// duplicated, and not guaranteed to be ordered in any way.
+  void collectVisibleTopLevelModuleNames(
+      SmallVectorImpl<Identifier> &names) const override;
+  ~ExplicitSwiftModuleLoader();
+};
+
 struct ModuleInterfaceLoaderOptions {
   bool remarkOnRebuildFromInterface = false;
   bool disableInterfaceLock = false;
@@ -137,6 +169,7 @@ struct ModuleInterfaceLoaderOptions {
     disableImplicitSwiftModule(Opts.DisableImplicitModules) {}
   ModuleInterfaceLoaderOptions() = default;
 };
+
 /// A ModuleLoader that runs a subordinate \c CompilerInvocation and
 /// \c CompilerInstance to convert .swiftinterface files to .swiftmodule
 /// files on the fly, caching the resulting .swiftmodules in the module cache

include/swift/Serialization/SerializedModuleLoader.h

@@ -138,6 +138,9 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// Scan the given serialized module file to determine dependencies.
   llvm::ErrorOr<ModuleDependencies> scanModuleFile(Twine modulePath);
 
+  /// Load the module file into a buffer and also collect its module name.
+  static std::unique_ptr<llvm::MemoryBuffer>
+  getModuleName(ASTContext &Ctx, StringRef modulePath, std::string &Name);
 public:
   virtual ~SerializedModuleLoaderBase();
   SerializedModuleLoaderBase(const SerializedModuleLoaderBase &) = delete;

lib/Frontend/Frontend.cpp

@@ -450,6 +450,16 @@ bool CompilerInstance::setUpModuleLoaders() {
     return true;
   }
 
+  // If implicit modules are disabled, we need to install an explicit module
+  // loader.
+  if (Invocation.getFrontendOptions().DisableImplicitModules) {
+    auto ESML = ExplicitSwiftModuleLoader::create(
+        *Context,
+        getDependencyTracker(), MLM,
+        Invocation.getSearchPathOptions().ExplicitSwiftModules,
+        IgnoreSourceInfoFile);
+    Context->addModuleLoader(std::move(ESML));
+  }
   if (MLM != ModuleLoadingMode::OnlySerialized) {
     auto const &Clang = clangImporter->getClangInstance();
     std::string ModuleCachePath = getModuleCachePathFromClang(Clang);

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -880,6 +880,10 @@ class ModuleInterfaceLoaderImpl {
 
       return std::move(module.moduleBuffer);
     }
+    // If implicit module is disabled, we are done.
+    if (Opts.disableImplicitSwiftModule) {
+      return std::make_error_code(std::errc::not_supported);
+    }
 
     std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
 
@@ -1397,3 +1401,81 @@ bool InterfaceSubContextDelegateImpl::runInSubCompilerInstance(StringRef moduleN
   // Run the action under the sub compiler instance.
   return action(info);
 }
+
+struct ExplicitSwiftModuleLoader::Implementation {
+  // Information about explicitly specified Swift module files.
+  struct ExplicitModuleInfo {
+    // Path of the module file.
+    StringRef path;
+    // Buffer of the module content.
+    std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
+  };
+  llvm::StringMap<ExplicitModuleInfo> ExplicitModuleMap;
+};
+
+ExplicitSwiftModuleLoader::ExplicitSwiftModuleLoader(
+      ASTContext &ctx,
+      DependencyTracker *tracker,
+      ModuleLoadingMode loadMode,
+      bool IgnoreSwiftSourceInfoFile):
+        SerializedModuleLoaderBase(ctx, tracker, loadMode,
+                                   IgnoreSwiftSourceInfoFile),
+        Impl(*new Implementation()) {}
+
+ExplicitSwiftModuleLoader::~ExplicitSwiftModuleLoader() { delete &Impl; }
+
+std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
+    AccessPathElem ModuleID,
+    const SerializedModuleBaseName &BaseName,
+    SmallVectorImpl<char> *ModuleInterfacePath,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
+    std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer) {
+  StringRef moduleName = ModuleID.Item.str();
+  auto it = Impl.ExplicitModuleMap.find(moduleName);
+  // If no explicit module path is given matches the name, return with an
+  // error code.
+  if (it == Impl.ExplicitModuleMap.end()) {
+    return std::make_error_code(std::errc::not_supported);
+  }
+  // We found an explicit module matches the given name, give the buffer
+  // back to the caller side.
+  *ModuleBuffer = std::move(it->getValue().moduleBuffer);
+  return std::error_code();
+}
+
+void ExplicitSwiftModuleLoader::collectVisibleTopLevelModuleNames(
+      SmallVectorImpl<Identifier> &names) const {
+  for (auto &entry: Impl.ExplicitModuleMap) {
+    names.push_back(Ctx.getIdentifier(entry.getKey()));
+  }
+}
+
+std::unique_ptr<ExplicitSwiftModuleLoader>
+ExplicitSwiftModuleLoader::create(ASTContext &ctx,
+    DependencyTracker *tracker, ModuleLoadingMode loadMode,
+    ArrayRef<std::string> ExplicitModulePaths,
+    bool IgnoreSwiftSourceInfoFile) {
+  auto result = std::unique_ptr<ExplicitSwiftModuleLoader>(
+    new ExplicitSwiftModuleLoader(ctx, tracker, loadMode,
+                                  IgnoreSwiftSourceInfoFile));
+  auto &Impl = result->Impl;
+  for (auto path: ExplicitModulePaths) {
+    std::string name;
+    // Load the explicit module into a buffer and get its name.
+    std::unique_ptr<llvm::MemoryBuffer> buffer = getModuleName(ctx, path, name);
+    if (buffer) {
+      // Register this module for future loading.
+      auto &entry = Impl.ExplicitModuleMap[name];
+      entry.path = path;
+      entry.moduleBuffer = std::move(buffer);
+    } else {
+      // We cannot read the module content, diagnose.
+      ctx.Diags.diagnose(SourceLoc(),
+                         diag::error_opening_explicit_module_file,
+                         path);
+    }
+  }
+
+  return result;
+}

lib/Serialization/ModuleFile.cpp

@@ -2190,6 +2190,30 @@ TypeDecl *ModuleFile::lookupLocalType(StringRef MangledName) {
   return cast<TypeDecl>(getDecl(*iter));
 }
 
+std::unique_ptr<llvm::MemoryBuffer>
+ModuleFile::getModuleName(ASTContext &Ctx, StringRef modulePath,
+                          std::string &Name) {
+  // Open the module file
+  auto &fs = *Ctx.SourceMgr.getFileSystem();
+  auto moduleBuf = fs.getBufferForFile(modulePath);
+  if (!moduleBuf)
+    return nullptr;
+
+  // Load the module file without validation.
+  std::unique_ptr<ModuleFile> loadedModuleFile;
+  ExtendedValidationInfo ExtInfo;
+  bool isFramework = false;
+  serialization::ValidationInfo loadInfo =
+     ModuleFile::load(modulePath.str(),
+                      std::move(moduleBuf.get()),
+                      nullptr,
+                      nullptr,
+                      /*isFramework*/isFramework, loadedModuleFile,
+                      &ExtInfo);
+  Name = loadedModuleFile->Name;
+  return std::move(loadedModuleFile->ModuleInputBuffer);
+}
+
 OpaqueTypeDecl *ModuleFile::lookupOpaqueResultType(StringRef MangledName) {
   PrettyStackTraceModuleFile stackEntry(*this);
 

lib/Serialization/ModuleFile.h

@@ -106,6 +106,10 @@ class ModuleFile
   StringRef MiscVersion;
 
 public:
+  static std::unique_ptr<llvm::MemoryBuffer> getModuleName(ASTContext &Ctx,
+                                                           StringRef modulePath,
+                                                           std::string &Name);
+
   /// Represents another module that has been imported as a dependency.
   class Dependency {
   public:

lib/Serialization/SerializedModuleLoader.cpp

@@ -275,6 +275,12 @@ std::error_code SerializedModuleLoaderBase::openModuleDocFileIfPresent(
   return std::error_code();
 }
 
+std::unique_ptr<llvm::MemoryBuffer>
+SerializedModuleLoaderBase::getModuleName(ASTContext &Ctx, StringRef modulePath,
+                                          std::string &Name) {
+  return ModuleFile::getModuleName(Ctx, modulePath, Name);
+}
+
 std::error_code
 SerializedModuleLoaderBase::openModuleSourceInfoFileIfPresent(
     AccessPathElem ModuleID,

test/ScanDependencies/Inputs/BuildModulesFromGraph.swift

@@ -22,7 +22,7 @@ let moduleDependencyGraph = try! decoder.decode(
 
 func findModuleBuildingCommand(_ moduleName: String) -> [String]? {
   for (_, dep) in moduleDependencyGraph.modules {
-    if dep.modulePath.hasSuffix(moduleName) {
+    if URL(fileURLWithPath: dep.modulePath).lastPathComponent == moduleName {
       switch dep.details {
       case .swift(let details):
         return details.commandLine

test/ScanDependencies/Inputs/Swift/SubE.swiftinterface

@@ -0,0 +1,5 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name SubE
+import Swift
+import E
+public func funcSubE() {}