Merge pull request #75181 from cachemeifyoucan/eng/PR-macro-deps-swift-6

[6.0][Macro] Get accurate dependency information for macro in explicit module build

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -46,6 +46,10 @@ typedef struct swiftscan_module_details_s *swiftscan_module_details_t;
 /// Opaque container to a dependency info of a given module.
 typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t;
 
+
+/// Opaque container to a macro dependency.
+typedef struct swiftscan_macro_dependency_s *swiftscan_macro_dependency_t;
+
 /// Opaque container to an overall result of a dependency scan.
 typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t;
 
@@ -64,6 +68,12 @@ typedef struct {
   size_t count;
 } swiftscan_dependency_set_t;
 
+/// Set of macro dependency
+typedef struct {
+  swiftscan_macro_dependency_t *macro_dependencies;
+  size_t count;
+} swiftscan_macro_dependency_set_t;
+
 typedef enum {
   SWIFTSCAN_DIAGNOSTIC_SEVERITY_ERROR = 0,
   SWIFTSCAN_DIAGNOSTIC_SEVERITY_WARNING = 1,

include/swift/AST/ModuleDependencies.h

@@ -88,6 +88,12 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
+/// This is used to idenfity a specific macro plugin dependency.
+struct MacroPluginDependency {
+  std::string LibraryPath;
+  std::string ExecutablePath;
+};
+
 /// This is used to identify a specific module.
 struct ModuleDependencyID {
   std::string ModuleName;
@@ -240,6 +246,9 @@ struct CommonSwiftTextualModuleDependencyDetails {
   /// (Clang) modules on which the bridging header depends.
   std::vector<std::string> bridgingModuleDependencies;
 
+  /// The macro dependencies.
+  std::map<std::string, MacroPluginDependency> macroDependencies;
+
   /// The Swift frontend invocation arguments to build the Swift module from the
   /// interface.
   std::vector<std::string> buildCommandLine;
@@ -302,6 +311,12 @@ class SwiftInterfaceModuleDependenciesStorage :
   void updateCommandLine(const std::vector<std::string> &newCommandLine) {
     textualModuleDetails.buildCommandLine = newCommandLine;
   }
+
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {macroModuleName.str(), {libraryPath.str(), executablePath.str()}});
+  }
 };
 
 /// Describes the dependencies of a Swift module
@@ -353,6 +368,12 @@ class SwiftSourceModuleDependenciesStorage :
   void addTestableImport(ImportPath::Module module) {
     testableImports.insert(module.front().Item.str());
   }
+
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath) {
+    textualModuleDetails.macroDependencies.insert(
+        {macroModuleName.str(), {libraryPath.str(), executablePath.str()}});
+  }
 };
 
 /// Describes the dependencies of a pre-built Swift module (with no
@@ -758,6 +779,10 @@ class ModuleDependencyInfo {
   /// For a Source dependency, register a `Testable` import
   void addTestableImport(ImportPath::Module module);
 
+  /// For a Source/Textual dependency, register a macro dependency.
+  void addMacroDependency(StringRef macroModuleName, StringRef libraryPath,
+                          StringRef executablePath);
+
   /// Whether or not a queried module name is a `@Testable` import dependency
   /// of this module. Can only return `true` for Swift source modules.
   bool isTestableImport(StringRef moduleName) const;

include/swift/DependencyScan/DependencyScanImpl.h

@@ -64,6 +64,12 @@ struct swiftscan_dependency_info_s {
   swiftscan_module_details_t details;
 };
 
+struct swiftscan_macro_dependency_s {
+  swiftscan_string_ref_t moduleName;
+  swiftscan_string_ref_t libraryPath;
+  swiftscan_string_ref_t executablePath;
+};
+
 /// Swift modules to be built from a module interface, may have a bridging
 /// header.
 typedef struct {
@@ -111,6 +117,9 @@ typedef struct {
 
   /// ModuleCacheKey
   swiftscan_string_ref_t module_cache_key;
+
+  /// Macro dependecies.
+  swiftscan_macro_dependency_set_t *macro_dependencies;
 } swiftscan_swift_textual_details_t;
 
 /// Swift modules with only a binary module file.

lib/AST/ModuleDependencies.cpp

@@ -17,6 +17,8 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/DiagnosticsSema.h"
+#include "swift/AST/MacroDefinition.h"
+#include "swift/AST/PluginLoader.h"
 #include "swift/AST/SourceFile.h"
 #include "swift/Frontend/Frontend.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
@@ -103,6 +105,21 @@ void ModuleDependencyInfo::addTestableImport(ImportPath::Module module) {
   dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get())->addTestableImport(module);
 }
 
+void ModuleDependencyInfo::addMacroDependency(StringRef macroModuleName,
+                                              StringRef libraryPath,
+                                              StringRef executablePath) {
+  if (auto swiftSourceStorage =
+          dyn_cast<SwiftSourceModuleDependenciesStorage>(storage.get()))
+    swiftSourceStorage->addMacroDependency(macroModuleName, libraryPath,
+                                           executablePath);
+  else if (auto swiftInterfaceStorage =
+               dyn_cast<SwiftInterfaceModuleDependenciesStorage>(storage.get()))
+    swiftInterfaceStorage->addMacroDependency(macroModuleName, libraryPath,
+                                              executablePath);
+  else
+    llvm_unreachable("Unexpected dependency kind");
+}
+
 bool ModuleDependencyInfo::isTestableImport(StringRef moduleName) const {
   if (auto swiftSourceDepStorage = getAsSwiftSourceModule())
     return swiftSourceDepStorage->testableImports.contains(moduleName);
@@ -183,35 +200,45 @@ void ModuleDependencyInfo::addModuleImports(
   SmallVector<Decl *, 32> decls;
   sourceFile.getTopLevelDecls(decls);
   for (auto decl : decls) {
-    auto importDecl = dyn_cast<ImportDecl>(decl);
-    if (!importDecl)
-      continue;
-
-    ImportPath::Builder scratch;
-    auto realPath = importDecl->getRealModulePath(scratch);
-
-    // Explicit 'Builtin' import is not a part of the module's
-    // dependency set, does not exist on the filesystem,
-    // and is resolved within the compiler during compilation.
-    SmallString<64> importedModuleName;
-    realPath.getString(importedModuleName);
-    if (importedModuleName == BUILTIN_NAME)
-      continue;
-
-    // Ignore/diagnose tautological imports akin to import resolution
-    if (!swift::dependencies::checkImportNotTautological(
-            realPath, importDecl->getLoc(), sourceFile,
-            importDecl->isExported()))
-      continue;
+    if (auto importDecl = dyn_cast<ImportDecl>(decl)) {
+      ImportPath::Builder scratch;
+      auto realPath = importDecl->getRealModulePath(scratch);
+
+      // Explicit 'Builtin' import is not a part of the module's
+      // dependency set, does not exist on the filesystem,
+      // and is resolved within the compiler during compilation.
+      SmallString<64> importedModuleName;
+      realPath.getString(importedModuleName);
+      if (importedModuleName == BUILTIN_NAME)
+        continue;
 
-    addModuleImport(realPath, &alreadyAddedModules,
-                    sourceManager, importDecl->getLoc());
+      // Ignore/diagnose tautological imports akin to import resolution
+      if (!swift::dependencies::checkImportNotTautological(
+              realPath, importDecl->getLoc(), sourceFile,
+              importDecl->isExported()))
+        continue;
 
-    // Additionally, keep track of which dependencies of a Source
-    // module are `@Testable`.
-    if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
-        importDecl->isTestable())
-      addTestableImport(realPath);
+      addModuleImport(realPath, &alreadyAddedModules, sourceManager,
+                      importDecl->getLoc());
+
+      // Additionally, keep track of which dependencies of a Source
+      // module are `@Testable`.
+      if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
+          importDecl->isTestable())
+        addTestableImport(realPath);
+    } else if (auto macroDecl = dyn_cast<MacroDecl>(decl)) {
+      auto macroDef = macroDecl->getDefinition();
+      auto &ctx = macroDecl->getASTContext();
+      if (macroDef.kind != MacroDefinition::Kind::External)
+        continue;
+      auto external = macroDef.getExternalMacro();
+      PluginLoader &loader = ctx.getPluginLoader();
+      auto &entry = loader.lookupPluginByModuleName(external.moduleName);
+      if (entry.libraryPath.empty() && entry.executablePath.empty())
+        continue;
+      addMacroDependency(external.moduleName.str(), entry.libraryPath,
+                         entry.executablePath);
+    }
   }
 
   auto fileName = sourceFile.getFilename();
@@ -535,58 +562,6 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
   // Add VFSOverlay file.
   for (auto &Overlay: Opts.VFSOverlayFiles)
     FS->status(Overlay);
-
-  // Add plugin dylibs from the toolchain only by look through the plugin search
-  // directory.
-  auto recordFiles = [&](StringRef Path) {
-    std::error_code EC;
-    for (auto I = FS->dir_begin(Path, EC);
-         !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
-      if (I->type() != llvm::sys::fs::file_type::regular_file)
-        continue;
-#if defined(_WIN32)
-      constexpr StringRef libPrefix{};
-      constexpr StringRef libSuffix = ".dll";
-#else
-      constexpr StringRef libPrefix = "lib";
-      constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
-#endif
-      StringRef filename = llvm::sys::path::filename(I->path());
-      if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
-        FS->status(I->path());
-    }
-  };
-  for (auto &entry : Opts.PluginSearchOpts) {
-    switch (entry.getKind()) {
-
-    // '-load-plugin-library <library path>'.
-    case PluginSearchOption::Kind::LoadPluginLibrary: {
-      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
-      FS->status(val.LibraryPath);
-      break;
-    }
-
-    // '-load-plugin-executable <executable path>#<module name>, ...'.
-    case PluginSearchOption::Kind::LoadPluginExecutable: {
-      // We don't have executable plugin in toolchain.
-      break;
-    }
-
-    // '-plugin-path <library search path>'.
-    case PluginSearchOption::Kind::PluginPath: {
-      auto &val = entry.get<PluginSearchOption::PluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-
-    // '-external-plugin-path <library search path>#<server path>'.
-    case PluginSearchOption::Kind::ExternalPluginPath: {
-      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
-      recordFiles(val.SearchPath);
-      break;
-    }
-    }
-  }
 }
 
 void SwiftDependencyTracker::startTracking() {

lib/DependencyScan/DependencyScanJSON.cpp

@@ -202,6 +202,49 @@ static void writeDependencies(llvm::raw_ostream &out,
   out << "\n";
 }
 
+static void
+writeMacroDependencies(llvm::raw_ostream &out,
+                       const swiftscan_macro_dependency_set_t *macro_deps,
+                       unsigned indentLevel, bool trailingComma) {
+  if (!macro_deps)
+    return;
+
+  out.indent(indentLevel * 2);
+  out << "\"macroDependencies\": ";
+  out << "[\n";
+  for (size_t i = 0; i < macro_deps->count; ++i) {
+    const auto &macroInfo = *macro_deps->macro_dependencies[i];
+    out.indent((indentLevel + 1) * 2);
+    out << "{\n";
+    auto entryIndentLevel = ((indentLevel + 2) * 2);
+    out.indent(entryIndentLevel);
+    out << "\"moduleName\": ";
+    writeJSONValue(out, macroInfo.moduleName, indentLevel);
+    out << ",\n";
+    out.indent(entryIndentLevel);
+    out << "\"libraryPath\": ";
+    writeJSONValue(out, macroInfo.libraryPath, entryIndentLevel);
+    out << ",\n";
+    out.indent(entryIndentLevel);
+    out << "\"executablePath\": ";
+    writeJSONValue(out, macroInfo.executablePath, entryIndentLevel);
+    out << "\n";
+    out.indent((indentLevel + 1) * 2);
+    out << "}";
+    if (i != macro_deps->count - 1) {
+      out << ",";
+    }
+    out << "\n";
+  }
+
+  out.indent(indentLevel * 2);
+  out << "]";
+
+  if (trailingComma)
+    out << ",";
+  out << "\n";
+}
+
 static const swiftscan_swift_textual_details_t *
 getAsTextualDependencyModule(swiftscan_module_details_t details) {
   if (details->kind == SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL)
@@ -369,6 +412,8 @@ void writeJSON(llvm::raw_ostream &out,
                              swiftTextualDeps->module_cache_key, 5,
                              /*trailingComma=*/true);
       }
+      writeMacroDependencies(out, swiftTextualDeps->macro_dependencies, 5,
+                             /*trailingComma=*/true);
       writeJSONSingleField(out, "isFramework", swiftTextualDeps->is_framework,
                            5, commaAfterFramework);
       if (swiftTextualDeps->extra_pcm_args->count != 0) {

lib/DependencyScan/DependencyScanningTool.cpp

@@ -227,7 +227,8 @@ static swiftscan_dependency_graph_t generateHollowDiagnosticOutput(
                                           false,
                                           c_string_utils::create_null(),
                                           c_string_utils::create_null(),
-                                          c_string_utils::create_null()};
+                                          c_string_utils::create_null(),
+                                          nullptr};
   hollowMainModuleInfo->details = hollowDetails;
 
   // Populate the diagnostic info