[Caching] Preliminary simple macro support for caching

Preliminary caching support for macro:
* Inserting the plugin into the CASFS
* Lookup plugin via physical file system

For future better support, we should teach dependency scanner to resolve
macros and return the resolved plugins to swift-frontend.

rdar://121873571

Author: cachemeifyoucan

lib/AST/ModuleDependencies.cpp

@@ -21,6 +21,7 @@
 #include "swift/Frontend/Frontend.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrefixMapper.h"
@@ -481,6 +482,58 @@ void SwiftDependencyTracker::addCommonSearchPathDeps(
   // Add VFSOverlay file.
   for (auto &Overlay: Opts.VFSOverlayFiles)
     FS->status(Overlay);
+
+  // Add plugin dylibs from the toolchain only by look through the plugin search
+  // directory.
+  auto recordFiles = [&](StringRef Path) {
+    std::error_code EC;
+    for (auto I = FS->dir_begin(Path, EC);
+         !EC && I != llvm::vfs::directory_iterator(); I = I.increment(EC)) {
+      if (I->type() != llvm::sys::fs::file_type::regular_file)
+        continue;
+#if defined(_WIN32)
+      constexpr StringRef libPrefix{};
+      constexpr StringRef libSuffix = ".dll";
+#else
+      constexpr StringRef libPrefix = "lib";
+      constexpr StringRef libSuffix = LTDL_SHLIB_EXT;
+#endif
+      StringRef filename = llvm::sys::path::filename(I->path());
+      if (filename.starts_with(libPrefix) && filename.ends_with(libSuffix))
+        FS->status(I->path());
+    }
+  };
+  for (auto &entry : Opts.PluginSearchOpts) {
+    switch (entry.getKind()) {
+
+    // '-load-plugin-library <library path>'.
+    case PluginSearchOption::Kind::LoadPluginLibrary: {
+      auto &val = entry.get<PluginSearchOption::LoadPluginLibrary>();
+      FS->status(val.LibraryPath);
+      break;
+    }
+
+    // '-load-plugin-executable <executable path>#<module name>, ...'.
+    case PluginSearchOption::Kind::LoadPluginExecutable: {
+      // We don't have executable plugin in toolchain.
+      break;
+    }
+
+    // '-plugin-path <library search path>'.
+    case PluginSearchOption::Kind::PluginPath: {
+      auto &val = entry.get<PluginSearchOption::PluginPath>();
+      recordFiles(val.SearchPath);
+      break;
+    }
+
+    // '-external-plugin-path <library search path>#<server path>'.
+    case PluginSearchOption::Kind::ExternalPluginPath: {
+      auto &val = entry.get<PluginSearchOption::ExternalPluginPath>();
+      recordFiles(val.SearchPath);
+      break;
+    }
+    }
+  }
 }
 
 void SwiftDependencyTracker::startTracking() {

lib/AST/PluginLoader.cpp

@@ -17,6 +17,7 @@
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/Lexer.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/VirtualFileSystem.h"
 
 using namespace swift;
 
@@ -59,6 +60,15 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {
   return "";
 }
 
+static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+getPluginLoadingFS(ASTContext &Ctx) {
+  // If there is a clang include tree FS, using real file system to load plugin
+  // as the FS in SourceMgr doesn't support directory iterator.
+  if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
+    return llvm::vfs::getRealFileSystem();
+  return Ctx.SourceMgr.getFileSystem();
+}
+
 llvm::DenseMap<Identifier, PluginLoader::PluginEntry> &
 PluginLoader::getPluginMap() {
   if (PluginMap.has_value()) {
@@ -86,7 +96,7 @@ PluginLoader::getPluginMap() {
     (void)result;
   };
 
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   std::error_code ec;
 
   for (auto &entry : Ctx.SearchPathOpts.PluginSearchOpts) {
@@ -162,7 +172,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) {
 
 llvm::Expected<LoadedLibraryPlugin *>
 PluginLoader::loadLibraryPlugin(StringRef path) {
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   SmallString<128> resolvedPath;
   if (auto err = fs->getRealPath(path, resolvedPath)) {
     return llvm::createStringError(err, err.message());
@@ -186,7 +196,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) {
 
 llvm::Expected<LoadedExecutablePlugin *>
 PluginLoader::loadExecutablePlugin(StringRef path) {
-  auto fs = Ctx.SourceMgr.getFileSystem();
+  auto fs = getPluginLoadingFS(Ctx);
   SmallString<128> resolvedPath;
   if (auto err = fs->getRealPath(path, resolvedPath)) {
     return llvm::createStringError(err, err.message());

lib/Sema/TypeCheckMacros.cpp

@@ -290,7 +290,9 @@ initializeExecutablePlugin(ASTContext &ctx,
   if (!libraryPath.empty()) {
 #if SWIFT_BUILD_SWIFT_SYNTAX
     llvm::SmallString<128> resolvedLibraryPath;
-    auto fs = ctx.SourceMgr.getFileSystem();
+    auto fs = ctx.ClangImporterOpts.HasClangIncludeTreeRoot
+                  ? llvm::vfs::getRealFileSystem()
+                  : ctx.SourceMgr.getFileSystem();
     if (auto err = fs->getRealPath(libraryPath, resolvedLibraryPath)) {
       return llvm::createStringError(err, err.message());
     }

test/CAS/macro_option_set.swift

@@ -0,0 +1,53 @@
+// REQUIRES: swift_swift_parser
+
+// RUN: %empty-directory(%t)
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -plugin-path %swift-plugin-dir
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
+// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
+
+// FS: SwiftMacros
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
+// RUN: %swift_frontend_plain @%t/SwiftShims.cmd
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %target-swift-frontend \
+// RUN:   -typecheck -verify -cache-compile-job -cas-path %t/cas \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -plugin-path %swift-plugin-dir \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   %s @%t/MyApp.cmd
+
+import Swift
+
+@attached(member, names: named(RawValue), named(rawValue), named(`init`), arbitrary)
+@attached(extension, conformances: OptionSet)
+public macro OptionSet<RawType>() =
+  #externalMacro(module: "SwiftMacros", type: "OptionSetMacro")
+
+@OptionSet<UInt8>
+struct ShippingOptions {
+  private enum Options: Int {
+    case nextDay
+    case secondDay
+    case priority
+    case standard
+  }
+
+  static let express: ShippingOptions = [.nextDay, .secondDay]
+  static let all: ShippingOptions = [.express, .priority, .standard]
+}
+
+let options = ShippingOptions.express
+assert(options.contains(.nextDay))
+assert(options.contains(.secondDay))
+assert(!options.contains(.standard))
+

test/CAS/macro_plugin.swift

@@ -0,0 +1,73 @@
+// REQUIRES: swift_swift_parser
+
+/// Test loading and external library through `-load-plugin-library`
+/// TODO: switch this test case to use `-external-plugin-path`.
+
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/plugins)
+//
+//== Build the plugin library
+// RUN: %host-build-swift \
+// RUN:   -swift-version 5 \
+// RUN:   -emit-library \
+// RUN:   -o %t/plugins/%target-library-name(MacroDefinition) \
+// RUN:   -module-name=MacroDefinition \
+// RUN:   %S/../Macros/Inputs/syntax_macro_definitions.swift \
+// RUN:   -g -no-toolchain-stdlib-rpath
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -load-plugin-library %t/plugins/%target-library-name(MacroDefinition)
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
+// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
+
+// FS: MacroDefinition
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
+// RUN: %swift_frontend_plain @%t/SwiftShims.cmd
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %target-swift-frontend \
+// RUN:   -typecheck -verify -cache-compile-job -cas-path %t/cas \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -load-plugin-library %t/plugins/%target-library-name(MacroDefinition) \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   %s @%t/MyApp.cmd
+
+@attached(extension, conformances: P, names: named(requirement))
+macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")
+
+protocol P {
+  static func requirement()
+}
+
+struct Wrapped: P {
+  static func requirement() {
+    print("Wrapped.requirement")
+  }
+}
+
+@DelegatedConformance
+struct Generic<Element> {}
+
+// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Generic: P where Element: P {\n  static func requirement() {\n    Element.requirement()\n  }\n}"}}
+
+func requiresP(_ value: (some P).Type) {
+  value.requirement()
+}
+
+requiresP(Generic<Wrapped>.self)
+
+struct Outer {
+  @DelegatedConformance
+  struct Nested<Element> {}
+}
+
+// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Outer.Nested: P where Element: P {\n  static func requirement() {\n    Element.requirement()\n  }\n}"}}
+
+requiresP(Outer.Nested<Wrapped>.self)

test/CAS/macro_plugin_external.swift

@@ -0,0 +1,73 @@
+// REQUIRES: swift_swift_parser
+
+/// Test loading and external library through `-load-plugin-library`
+/// TODO: switch this test case to use `-external-plugin-path`.
+
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/plugins)
+//
+//== Build the plugin library
+// RUN: %host-build-swift \
+// RUN:   -swift-version 5 \
+// RUN:   -emit-library \
+// RUN:   -o %t/plugins/%target-library-name(MacroDefinition) \
+// RUN:   -module-name=MacroDefinition \
+// RUN:   %S/../Macros/Inputs/syntax_macro_definitions.swift \
+// RUN:   -g -no-toolchain-stdlib-rpath
+
+// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %s -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json MyApp casFSRootID > %t/fs.casid
+// RUN: llvm-cas -cas %t/cas -ls-tree-recursive @%t/fs.casid | %FileCheck %s --check-prefix=FS
+
+// FS: MacroDefinition
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/SwiftShims.cmd
+// RUN: %swift_frontend_plain @%t/SwiftShims.cmd
+
+// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps.json MyApp > %t/MyApp.cmd
+// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
+// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid
+
+// RUN: %target-swift-frontend \
+// RUN:   -typecheck -verify -cache-compile-job -cas-path %t/cas \
+// RUN:   -swift-version 5 -disable-implicit-swift-modules \
+// RUN:   -external-plugin-path %t/plugins/#%swift-plugin-server \
+// RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \
+// RUN:   %s @%t/MyApp.cmd
+
+@attached(extension, conformances: P, names: named(requirement))
+macro DelegatedConformance() = #externalMacro(module: "MacroDefinition", type: "DelegatedConformanceViaExtensionMacro")
+
+protocol P {
+  static func requirement()
+}
+
+struct Wrapped: P {
+  static func requirement() {
+    print("Wrapped.requirement")
+  }
+}
+
+@DelegatedConformance
+struct Generic<Element> {}
+
+// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Generic: P where Element: P {\n  static func requirement() {\n    Element.requirement()\n  }\n}"}}
+
+func requiresP(_ value: (some P).Type) {
+  value.requirement()
+}
+
+requiresP(Generic<Wrapped>.self)
+
+struct Outer {
+  @DelegatedConformance
+  struct Nested<Element> {}
+}
+
+// CHECK: {"expandMacroResult":{"diagnostics":[],"expandedSource":"extension Outer.Nested: P where Element: P {\n  static func requirement() {\n    Element.requirement()\n  }\n}"}}
+
+requiresP(Outer.Nested<Wrapped>.self)