[Macros] Add swift-plugin-server executable

This executable is intended to be installed in the toolchain and act as
an executable compiler plugin just like other 'macro' plugins.

This plugin server has an optional method 'loadPluginLibrary' that
dynamically loads dylib plugins.
The compiler has a newly added option '-external-plugin-path'. This
option receives a pair of the plugin library search path (just like
'-plugin-path') and the corresponding "plugin server" path, separated
by '#'. i.e.

  -external-plugin-path
    <plugin library search path>#<plugin server executable path>

For exmaple, when there's a macro decl:

  @freestanding(expression)
  macro stringify<T>(T) -> (T, String) =
      #externalMacro(module: "BasicMacro", type: "StringifyMacro")

The compiler look for 'libBasicMacro.dylib' in '-plugin-path' paths,
if not found, it falls back to '-external-plugin-path' and tries to find
'libBasicMacro.dylib' in them. If it's found, the "plugin server" path
is launched just like an executable plugin, then 'loadPluginLibrary'
method is invoked via IPC, which 'dlopen' the library path in the plugin
server. At the actual macro expansion, the mangled name for
'BasicMacro.StringifyMacro' is used to resolve the macro  just like
dylib plugins in the compiler.

This is useful for
 * Isolating the plugin process, so the plugin crashes doesn't result
   the compiler crash
 * Being able to use library plugins linked with other `swift-syntax`
   versions

rdar://105104850

Author: rintaro

include/swift/AST/ASTContext.h

@@ -1464,8 +1464,24 @@ class ASTContext final {
 
   Type getNamedSwiftType(ModuleDecl *module, StringRef name);
 
-  LoadedExecutablePlugin *
-  lookupExecutablePluginByModuleName(Identifier moduleName);
+  /// Lookup an executable plugin that is declared to handle \p moduleName
+  /// module by '-load-plugin-executable'. Note that the returned path might be
+  /// in the current VFS. i.e. use FS.getRealPath() to get the real path.
+  Optional<StringRef> lookupExecutablePluginByModuleName(Identifier moduleName);
+
+  /// From paths '-external-plugin-path', look for dylib file that has
+  /// 'lib${moduleName}.dylib' (or equialent depending on the platform) and
+  /// return the found dylib path and the path to the "plugin server" for that.
+  /// Note that the returned path might be in the current VFS.
+  Optional<std::pair<std::string, std::string>>
+  lookupExternalLibraryPluginByModuleName(Identifier moduleName);
+
+  /// Launch the specified executable plugin path resolving the path with the
+  /// current VFS. If it fails to load the plugin, a diagnostic is emitted, and
+  /// returns a nullptr.
+  /// NOTE: This method is idempotent. If the plugin is already loaded, the same
+  /// instance is simply returned.
+  LoadedExecutablePlugin *loadExecutablePlugin(StringRef path);
 
   /// Get the plugin registry this ASTContext is using.
   PluginRegistry *getPluginRegistry() const;

include/swift/AST/SearchPathOptions.h

@@ -382,6 +382,11 @@ class SearchPathOptions {
   /// macro implementations.
   std::vector<std::string> PluginSearchPaths;
 
+  /// Paths that contain compiler plugins and the path to the plugin server
+  /// executable.
+  /// e.g. '/path/to/usr/lib/swift/host/plugins#/path/to/usr/bin/plugin-server'.
+  std::vector<std::string> ExternalPluginSearchPaths;
+
   /// Don't look in for compiler-provided modules.
   bool SkipRuntimeLibraryImportPaths = false;
 

include/swift/Demangling/Demangle.h

@@ -725,6 +725,12 @@ bool isFunctionAttr(Node::Kind kind);
 /// contain symbolic references.
 llvm::StringRef makeSymbolicMangledNameStringRef(const char *base);
 
+/// Produce the mangled name for the nominal type descriptor of a type
+/// referenced by its module and type name.
+std::string mangledNameForTypeMetadataAccessor(llvm::StringRef moduleName,
+                                               llvm::StringRef typeName,
+                                               Node::Kind typeKind);
+
 SWIFT_END_INLINE_NAMESPACE
 } // end namespace Demangle
 } // end namespace swift

include/swift/Option/Options.td

@@ -303,6 +303,11 @@ def plugin_path : Separate<["-"], "plugin-path">,
   Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
   HelpText<"Add directory to the plugin search path">;
 
+def external_plugin_path : Separate<["-"], "external-plugin-path">,
+  Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
+  HelpText<"Add directory to the plugin search path with a plugin server executable">,
+  MetaVarName<"<path>#<plugin-server-path>">;
+
 def import_underlying_module : Flag<["-"], "import-underlying-module">,
   Flags<[FrontendOption, NoInteractiveOption]>,
   HelpText<"Implicitly imports the Objective-C half of a module">;

lib/AST/ASTContext.cpp

@@ -67,6 +67,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Config/config.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
@@ -6344,15 +6345,34 @@ Type ASTContext::getNamedSwiftType(ModuleDecl *module, StringRef name) {
   return decl->getDeclaredInterfaceType();
 }
 
-LoadedExecutablePlugin *
+Optional<StringRef>
 ASTContext::lookupExecutablePluginByModuleName(Identifier moduleName) {
   auto &execPluginPaths = getImpl().ExecutablePluginPaths;
   auto found = execPluginPaths.find(moduleName);
   if (found == execPluginPaths.end())
-    return nullptr;
+    return None;
+  return found->second;
+}
+
+Optional<std::pair<std::string, std::string>>
+ASTContext::lookupExternalLibraryPluginByModuleName(Identifier moduleName) {
+  auto fs = this->SourceMgr.getFileSystem();
+  for (auto &pair : SearchPathOpts.ExternalPluginSearchPaths) {
+    StringRef searchPath;
+    StringRef serverPath;
+    std::tie(searchPath, serverPath) = StringRef(pair).split('#');
+
+    SmallString<128> fullPath(searchPath);
+    llvm::sys::path::append(fullPath, "lib" + moduleName.str() + LTDL_SHLIB_EXT);
+
+    if (fs->exists(fullPath)) {
+      return {{std::string(fullPath), serverPath.str()}};
+    }
+  }
+  return None;
+}
 
-  // Let the VFS to map the path.
-  auto &path = found->second;
+LoadedExecutablePlugin *ASTContext::loadExecutablePlugin(StringRef path) {
   SmallString<128> resolvedPath;
   auto fs = this->SourceMgr.getFileSystem();
   if (auto err = fs->getRealPath(path, resolvedPath)) {

lib/ASTGen/Package.swift

@@ -22,7 +22,8 @@ let package = Package(
     .macOS(.v10_15)
   ],
   products: [
-    .library(name: "swiftASTGen", targets: ["swiftASTGen"])
+    .library(name: "swiftASTGen", targets: ["swiftASTGen"]),
+    .library(name: "swiftLLVMJSON", targets: ["swiftLLVMJSON"]),
   ],
   dependencies: [
     .package(path: "../../../swift-syntax")

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -112,6 +112,9 @@ public func resolveExecutableMacro(
   typeNameLength: Int,
   pluginOpaqueHandle: UnsafeMutableRawPointer
 ) -> UnsafeRawPointer {
+  // NOTE: This doesn't actually resolve anything.
+  // Executable plugins is "trusted" to have the macro implementation. If not,
+  // the actual expansion fails.
   let exportedPtr = UnsafeMutablePointer<ExportedExecutableMacro>.allocate(capacity: 1)
   exportedPtr.initialize(to: .init(
     moduleName: String(bufferStart: moduleName, count: moduleNameLength),

lib/ASTGen/Sources/ASTGen/PluginHost.swift

@@ -37,6 +37,35 @@ public func _deinitializePlugin(
   plugin.deinitialize()
 }
 
+/// Load the library plugin in the plugin server.
+@_cdecl("swift_ASTGen_pluginServerLoadLibraryPlugin")
+func swift_ASTGen_pluginServerLoadLibraryPlugin(
+  opaqueHandle: UnsafeMutableRawPointer,
+  libraryPath: UnsafePointer<Int8>,
+  moduleName: UnsafePointer<Int8>,
+  cxxDiagnosticEngine: UnsafeMutablePointer<UInt8>
+) -> Bool {
+  let plugin =  CompilerPlugin(opaqueHandle: opaqueHandle)
+  assert(plugin.capability.features?.contains("loadPluginLibrary") == true)
+  let libraryPath = String(cString: libraryPath)
+  let moduleName = String(cString: moduleName)
+  let diagEngine = PluginDiagnosticsEngine(cxxDiagnosticEngine: cxxDiagnosticEngine)
+
+  do {
+    let result = try plugin.sendMessageAndWait(
+      .loadPluginLibrary(libraryPath: libraryPath, moduleName: moduleName)
+    )
+    guard case .loadPluginLibraryResult(let loaded, let diagnostics) = result else {
+      throw PluginError.invalidReponseKind
+    }
+    diagEngine.emit(diagnostics);
+    return loaded
+  } catch {
+    diagEngine.diagnose(error: error)
+    return false
+  }
+}
+
 struct CompilerPlugin {
   let opaqueHandle: UnsafeMutableRawPointer
 
@@ -224,6 +253,14 @@ class PluginDiagnosticsEngine {
     }
   }
 
+  func diagnose(error: Error) {
+    self.emitSingle(
+      message: String(describing: error),
+      severity: .error,
+      position: .invalid
+    )
+  }
+
   /// Produce the C++ source location for a given position based on a
   /// syntax node.
   private func cxxSourceLocation(

lib/ASTGen/Sources/ASTGen/PluginMessages.swift

@@ -14,14 +14,17 @@
 // NOTE: Types in this file should be self-contained and should not depend on any non-stdlib types.
 
 internal enum HostToPluginMessage: Codable {
+  /// Get capability of this plugin.
   case getCapability
 
+  /// Expand a '@freestanding' macro.
   case expandFreestandingMacro(
     macro: PluginMessage.MacroReference,
     discriminator: String,
     syntax: PluginMessage.Syntax
   )
 
+  /// Expand an '@attached' macro.
   case expandAttachedMacro(
     macro: PluginMessage.MacroReference,
     macroRole: PluginMessage.MacroRole,
@@ -30,9 +33,21 @@ internal enum HostToPluginMessage: Codable {
     declSyntax: PluginMessage.Syntax,
     parentDeclSyntax: PluginMessage.Syntax?
   )
+
+  /// Optionally implemented message to load a dynamic link library.
+  /// 'moduleName' can be used as a hint indicating that the library
+  /// provides the specified module.
+  case loadPluginLibrary(
+    libraryPath: String,
+    moduleName: String
+  )
 }
 
 internal enum PluginToHostMessage: Codable {
+  case getCapabilityResult(
+    capability: PluginMessage.PluginCapability
+  )
+
   case expandFreestandingMacroResult(
     expandedSource: String?,
     diagnostics: [PluginMessage.Diagnostic]
@@ -43,18 +58,21 @@ internal enum PluginToHostMessage: Codable {
     diagnostics: [PluginMessage.Diagnostic]
   )
 
-  case getCapabilityResult(capability: PluginMessage.PluginCapability)
+  case loadPluginLibraryResult(
+    loaded: Bool,
+    diagnostics: [PluginMessage.Diagnostic]
+  )
 }
 
 /*namespace*/ internal enum PluginMessage {
-  static var PROTOCOL_VERSION_NUMBER: Int { 3 }  // Renamed 'customAttributeSyntax' to 'attributeSyntax'.
+  static var PROTOCOL_VERSION_NUMBER: Int { 4 }  // Added 'loadPluginLibrary'.
 
   struct PluginCapability: Codable {
     var protocolVersion: Int
-  }
 
-  static var capability: PluginCapability {
-    PluginCapability(protocolVersion: PluginMessage.PROTOCOL_VERSION_NUMBER)
+    /// Optional features this plugin provides.
+    ///  * resolveLibraryMacro: 'resolveLibraryMacro' message is implemented.
+    var features: [String]?
   }
 
   struct MacroReference: Codable {

lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift

@@ -24,7 +24,7 @@ extension String {
 public struct LLVMJSON {
   /// Encode an `Encodable` value to JSON data, and call `body` is the buffer.
   /// Note that the buffer is valid onlu in `body`.
-  public static func encoding<T: Encodable, R>(_ value: T, body: (UnsafeBufferPointer<Int8>) -> R) throws -> R {
+  public static func encoding<T: Encodable, R>(_ value: T, body: (UnsafeBufferPointer<Int8>) throws -> R) throws -> R {
     let valuePtr = JSON_newValue()
     defer { JSON_value_delete(valuePtr) }
 
@@ -36,7 +36,7 @@ public struct LLVMJSON {
     assert(data.baseAddress != nil)
     defer { BridgedData_free(data) }
     let buffer = UnsafeBufferPointer(start: data.baseAddress, count: data.size)
-    return body(buffer)
+    return try body(buffer)
   }
 
   /// Decode a JSON data to a Swift value.

lib/CMakeLists.txt

@@ -28,6 +28,7 @@ if (SWIFT_SWIFT_PARSER)
     SwiftOperators
     SwiftSyntaxBuilder
     SwiftSyntaxMacros
+    SwiftCompilerPluginMessageHandling
   )
 
   # Compute the list of SwiftSyntax targets that we will link against.

lib/Demangling/Demangler.cpp

@@ -306,6 +306,41 @@ bool swift::Demangle::isStruct(llvm::StringRef mangledName) {
   return isStructNode(Dem.demangleType(mangledName));
 }
 
+std::string swift::Demangle::mangledNameForTypeMetadataAccessor(
+    StringRef moduleName, StringRef typeName, Node::Kind typeKind) {
+  using namespace Demangle;
+
+  //  kind=Global
+  //    kind=NominalTypeDescriptor
+  //      kind=Type
+  //        kind=Structure|Enum|Class
+  //          kind=Module, text=moduleName
+  //          kind=Identifier, text=typeName
+  Demangle::Demangler D;
+  auto *global = D.createNode(Node::Kind::Global);
+  {
+    auto *nominalDescriptor =
+        D.createNode(Node::Kind::TypeMetadataAccessFunction);
+    {
+      auto *type = D.createNode(Node::Kind::Type);
+      {
+        auto *module = D.createNode(Node::Kind::Module, moduleName);
+        auto *identifier = D.createNode(Node::Kind::Identifier, typeName);
+        auto *structNode = D.createNode(typeKind);
+        structNode->addChild(module, D);
+        structNode->addChild(identifier, D);
+        type->addChild(structNode, D);
+      }
+      nominalDescriptor->addChild(type, D);
+    }
+    global->addChild(nominalDescriptor, D);
+  }
+
+  auto mangleResult = mangleNode(global);
+  assert(mangleResult.isSuccess());
+  return mangleResult.result();
+}
+
 using namespace swift;
 using namespace Demangle;
 

lib/Driver/ToolChains.cpp

@@ -237,6 +237,7 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI,
   inputArgs.AddAllArgs(arguments, options::OPT_F, options::OPT_Fsystem);
   inputArgs.AddAllArgs(arguments, options::OPT_vfsoverlay);
   inputArgs.AddAllArgs(arguments, options::OPT_plugin_path);
+  inputArgs.AddAllArgs(arguments, options::OPT_external_plugin_path);
 
   inputArgs.AddLastArg(arguments, options::OPT_AssertConfig);
   inputArgs.AddLastArg(arguments, options::OPT_autolink_force_load);

lib/Frontend/CompilerInvocation.cpp

@@ -1494,6 +1494,15 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts,
     Opts.PluginSearchPaths.push_back(resolveSearchPath(A->getValue()));
   }
 
+  for (const Arg *A : Args.filtered(OPT_external_plugin_path)) {
+    // '<plugin directory>#<plugin server executable path>'.
+    StringRef dylibPath;
+    StringRef serverPath;
+    std::tie(dylibPath, serverPath) = StringRef(A->getValue()).split('#');
+    Opts.ExternalPluginSearchPaths.push_back(
+        resolveSearchPath(dylibPath) + "#" + resolveSearchPath(serverPath));
+  }
+
   for (const Arg *A : Args.filtered(OPT_L)) {
     Opts.LibrarySearchPaths.push_back(resolveSearchPath(A->getValue()));
   }