[Macros] Introduce Macro AST

include/swift/AST/CompilerPlugin.h

@@ -52,6 +52,11 @@ class CompilerPlugin {
     GenericSignature = 4,
     // static func _typeSignature(...) -> (UnsafePointer<UInt8>, count: Int)
     TypeSignature = 5,
+    // static func _owningModule(...) -> (UnsafePointer<UInt8>, count: Int)
+    OwningModule = 6,
+    // static func _supplementalSignatureModules(...)
+    //     -> (UnsafePointer<UInt8>, count: Int)
+    SupplementalSignatureModules = 7,
   };
 
   /// The plugin type metadata.
@@ -100,6 +105,14 @@ class CompilerPlugin {
   /// result string buffer.
   StringRef invokeTypeSignature() const;
 
+  /// Invoke the `_owningModule` method. The caller assumes ownership of the
+  /// result string buffer.
+  StringRef invokeOwningModule() const;
+
+  /// Invoke the `_supplementalSignatureModules` method. The caller assumes
+  /// ownership of the result string buffer.
+  StringRef invokeSupplementalSignatureModules() const;
+
   StringRef getName() const {
     return name;
   }

include/swift/AST/Macro.h

@@ -0,0 +1,86 @@
+//===--- Macro.h - Swift Macro Definition -----------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the `Macro` type that describes a macro definition.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_MACRO_H
+#define SWIFT_AST_MACRO_H
+
+#include "swift/AST/ASTAllocated.h"
+#include "swift/AST/GenericSignature.h"
+#include "swift/AST/Identifier.h"
+#include "swift/AST/Type.h"
+
+namespace swift {
+  class ModuleDecl;
+
+  class Macro : public ASTAllocated<Macro> {
+  public:
+    /// The kind of macro, which determines how it can be used in source code.
+    enum Kind: uint8_t {
+      /// An expression macro.
+      Expression,
+    };
+
+    /// Describes how the macro is implemented.
+    enum class ImplementationKind: uint8_t {
+      /// The macro is built-in to the compiler, linked against the same
+      /// underlying syntax tree libraries.
+      Builtin,
+
+      /// The macro was defined in a compiler plugin.
+      Plugin,
+    };
+
+    /// The kind of macro.
+    const Kind kind;
+
+    /// How the macro is implemented.
+    const ImplementationKind implementationKind;
+
+    /// The name of the macro in the source, e.g., "stringify".
+    const Identifier name;
+
+    /// The generic signature, used to describe the signature of macros that
+    /// involve generic parameters.
+    const GenericSignature genericSignature;
+
+    /// The type signature of the macro.
+    const Type signature;
+
+    /// The module with which this macro is associated.
+    ModuleDecl * const owningModule;
+
+    /// Supplemental modules that should be imported when 
+    const ArrayRef<ModuleDecl *> supplementalSignatureModules;
+
+    /// An opaque handle to the representation of the macro.
+    void * const opaqueHandle;
+
+  public:
+    Macro(
+      Kind kind, ImplementationKind implementationKind, Identifier name,
+      GenericSignature genericSignature, Type signature,
+      ModuleDecl *owningModule,
+      ArrayRef<ModuleDecl *> supplementalSignatureModules,
+      void *opaqueHandle
+    ) : kind(kind), implementationKind(implementationKind), name(name),
+        genericSignature(genericSignature), signature(signature),
+        owningModule(owningModule),
+        supplementalSignatureModules(supplementalSignatureModules),
+        opaqueHandle(opaqueHandle) { }
+  };
+}
+
+#endif // SWIFT_AST_MACRO_H

include/swift/AST/TypeCheckRequests.h

@@ -45,6 +45,7 @@ class DefaultArgumentExpr;
 class DefaultArgumentType;
 class ClosureExpr;
 class GenericParamList;
+class Macro;
 class PrecedenceGroupDecl;
 class PropertyWrapperInitializerInfo;
 struct PropertyWrapperLValueness;
@@ -3733,35 +3734,20 @@ class SynthesizeHasSymbolQueryRequest
   bool isCached() const { return true; }
 };
 
-
-/// Retrieves the evaluation context of a macro with the given name.
-///
-/// The macro evaluation context is a user-defined generic signature and return
-/// type that serves as the "interface type" of references to the macro. The
-/// current implementation takes those pieces of syntax from the macro itself,
-/// then inserts them into a Swift struct that looks like
-///
-/// \code
-/// struct __MacroEvaluationContext\(macro.genericSignature) {
-///   typealias SignatureType = \(macro.signature)
-/// }
-/// \endcode
-///
-/// So that we can use all of Swift's native name lookup and type resolution
-/// facilities to map the parsed signature type back into a semantic \c Type
-/// AST and a set of requiremnets.
-class MacroContextRequest
-    : public SimpleRequest<MacroContextRequest,
-                           StructDecl *(std::string, ModuleDecl *),
+/// Lookup all macros with the given name that are visible from the given
+/// module.
+class MacroLookupRequest
+    : public SimpleRequest<MacroLookupRequest,
+                           ArrayRef<Macro *>(Identifier, ModuleDecl *),
                            RequestFlags::Cached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
-  StructDecl *evaluate(Evaluator &evaluator,
-                       std::string macroName, ModuleDecl *mod) const;
+  ArrayRef<Macro *> evaluate(Evaluator &evaluator,
+                             Identifier macroName, ModuleDecl *mod) const;
 
 public:
   bool isCached() const { return true; }
@@ -3772,6 +3758,7 @@ void simple_display(llvm::raw_ostream &out, Type value);
 void simple_display(llvm::raw_ostream &out, const TypeRepr *TyR);
 void simple_display(llvm::raw_ostream &out, ImplicitMemberAction action);
 void simple_display(llvm::raw_ostream &out, ResultBuilderBodyPreCheck pck);
+void simple_display(llvm::raw_ostream &out, const Macro *macro);
 
 #define SWIFT_TYPEID_ZONE TypeChecker
 #define SWIFT_TYPEID_HEADER "swift/AST/TypeCheckerTypeIDZone.def"

include/swift/AST/TypeCheckerTypeIDZone.def

@@ -440,6 +440,6 @@ SWIFT_REQUEST(TypeChecker, GetTypeWrapperInitializer,
 SWIFT_REQUEST(TypeChecker, SynthesizeHasSymbolQueryRequest,
               FuncDecl *(ValueDecl *),
               Cached, NoLocationInfo)
-SWIFT_REQUEST(TypeChecker, MacroContextRequest,
-              StructDecl *(std::string, ModuleDecl *),
+SWIFT_REQUEST(TypeChecker, MacroLookupRequest,
+              ArrayRef<Macro *>(Identifier, ModuleDecl *),
               Cached, NoLocationInfo)

include/swift/Sema/ConstraintSystem.h

@@ -4911,7 +4911,7 @@ class ConstraintSystem {
   ///
   /// \returns The opened type of the macro with this name, or the null \c Type
   /// if no such macro exists.
-  Type getTypeOfMacroReference(StringRef macro, Expr *anchor);
+  Type getTypeOfMacroReference(Identifier macroName, Expr *anchor);
 #endif
 
   /// Retrieve a list of generic parameter types solver has "opened" (replaced

lib/AST/CompilerPlugin.cpp

@@ -319,3 +319,27 @@ StringRef CompilerPlugin::invokeTypeSignature() const {
   llvm_unreachable("Incompatible host compiler");
 #endif
 }
+
+StringRef CompilerPlugin::invokeOwningModule() const {
+#if __clang__
+  using Method = SWIFT_CC CharBuffer(
+      SWIFT_CONTEXT const void *, const void *, const void *);
+  auto method = getWitnessMethodUnsafe<Method>(
+      WitnessTableEntry::OwningModule);
+  return method(metadata, metadata, witnessTable).str();
+#else
+  llvm_unreachable("Incompatible host compiler");
+#endif
+}
+
+StringRef CompilerPlugin::invokeSupplementalSignatureModules() const {
+#if __clang__
+  using Method = SWIFT_CC CharBuffer(
+      SWIFT_CONTEXT const void *, const void *, const void *);
+  auto method = getWitnessMethodUnsafe<Method>(
+      WitnessTableEntry::SupplementalSignatureModules);
+  return method(metadata, metadata, witnessTable).str();
+#else
+  llvm_unreachable("Incompatible host compiler");
+#endif
+}

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -1,3 +1,4 @@
+import SwiftParser
 import SwiftSyntax
 @_spi(Testing) import _SwiftSyntaxMacros
 
@@ -81,43 +82,107 @@ private func allocateUTF8String(
   }
 }
 
+@_cdecl("swift_ASTGen_getMacroGenericSignature")
+public func getMacroGenericSignature(
+  macroPtr: UnsafeMutablePointer<UInt8>,
+  genericSignaturePtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  genericSignatureLengthPtr: UnsafeMutablePointer<Int>
+) {
+  macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
+    guard let genericSig = macro.pointee.macro.genericSignature else {
+      (genericSignaturePtr.pointee, genericSignatureLengthPtr.pointee) = (nil, 0)
+      return
+    }
+
+    (genericSignaturePtr.pointee, genericSignatureLengthPtr.pointee) =
+      allocateUTF8String(genericSig.description)
+  }
+}
+
 /// Query the type signature of the given macro.
 @_cdecl("swift_ASTGen_getMacroTypeSignature")
 public func getMacroTypeSignature(
   macroPtr: UnsafeMutablePointer<UInt8>,
-  evaluationContextPtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
-  evaluationContextLengthPtr: UnsafeMutablePointer<Int>
+  signaturePtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  signatureLengthPtr: UnsafeMutablePointer<Int>
 ) {
   macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
-    (evaluationContextPtr.pointee, evaluationContextLengthPtr.pointee) =
-      allocateUTF8String(macro.pointee.evaluationContext, nullTerminated: true)
+    (signaturePtr.pointee, signatureLengthPtr.pointee) =
+        allocateUTF8String(macro.pointee.macro.signature.description)
   }
 }
 
-extension ExportedMacro {
-  var evaluationContext: String {
-    """
-    struct __MacroEvaluationContext\(self.macro.genericSignature?.description ?? "") {
-      typealias SignatureType = \(self.macro.signature)
-    }
-    """
+/// Query the documentation of the given macro.
+@_cdecl("swift_ASTGen_getMacroDocumentation")
+public func getMacroDocumentation(
+  macroPtr: UnsafeMutablePointer<UInt8>,
+  documentationPtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  documentationLengthPtr: UnsafeMutablePointer<Int>
+) {
+  macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
+    (documentationPtr.pointee, documentationLengthPtr.pointee) =
+        allocateUTF8String(macro.pointee.macro.documentation)
   }
 }
 
-/// Query the macro evaluation context of the given macro.
+/// Query the owning module of the given macro.
+@_cdecl("swift_ASTGen_getMacroOwningModule")
+public func getMacroOwningModule(
+  macroPtr: UnsafeMutablePointer<UInt8>,
+  owningModulePtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  owningModuleLengthPtr: UnsafeMutablePointer<Int>
+) {
+  macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
+    (owningModulePtr.pointee, owningModuleLengthPtr.pointee) =
+        allocateUTF8String(macro.pointee.macro.owningModule)
+  }
+}
+
+/// Query the supplemental signature modules of the given macro,
+/// as a semicolon-separated string
+@_cdecl("swift_ASTGen_getMacroSupplementalSignatureModules")
+public func getMacroSupplementableSignatureModules(
+  macroPtr: UnsafeMutablePointer<UInt8>,
+  modulesPtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  modulesLengthPtr: UnsafeMutablePointer<Int>
+) {
+  macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
+    let modules = macro.pointee.macro.supplementalSignatureModules
+        .joined(separator: ";")
+    (modulesPtr.pointee, modulesLengthPtr.pointee) =
+        allocateUTF8String(modules)
+  }
+}
+
+/// Query the macro evaluation context given the evaluation
+/// context sources.
 @_cdecl("swift_ASTGen_getMacroEvaluationContext")
 public func getMacroEvaluationContext(
   sourceFilePtr: UnsafePointer<UInt8>,
   declContext: UnsafeMutableRawPointer,
   context: UnsafeMutableRawPointer,
-  macroPtr: UnsafeMutablePointer<UInt8>,
-  contextPtr: UnsafeMutablePointer<UnsafeMutableRawPointer?>
-) {
-  contextPtr.pointee = macroPtr.withMemoryRebound(to: ExportedMacro.self, capacity: 1) { macro in
-    return ASTGenVisitor(ctx: context, base: sourceFilePtr, declContext: declContext)
-      .visit(StructDeclSyntax(stringLiteral: macro.pointee.evaluationContext))
-      .rawValue
+  evaluatedSourceBuffer: UnsafePointer<UInt8>,
+  evaluatedSourceBufferLength: Int
+) -> UnsafeMutableRawPointer? {
+  let evaluatedSource = UnsafeBufferPointer(
+      start: evaluatedSourceBuffer,
+      count: evaluatedSourceBufferLength
+  )
+  let sourceFile = Parser.parse(source: evaluatedSource)
+
+  // Dig out the top-level typealias declaration. That's all we'll
+  // parse.
+  guard let typealiasDecl = sourceFile.statements.first(
+    where: { item in item.item.is(TypealiasDeclSyntax.self)
+    })?.item.as(TypealiasDeclSyntax.self) else {
+    return nil
   }
+
+  // Parse and ASTGen that top-level declaration.
+  // FIXME: we need to emit diagnostics from this.
+  return ASTGenVisitor(
+    ctx: context, base: sourceFilePtr, declContext: declContext
+  ).visit(typealiasDecl).rawValue
 }
 
 @_cdecl("swift_ASTGen_evaluateMacro")

lib/CompilerPluginSupport/CompilerPluginSupport.swift

@@ -69,4 +69,20 @@ public protocol _CompilerPlugin {
   /// - Returns: A newly allocated buffer containing the type signature. The
   ///   caller is responsible for managing the memory.
   static func _typeSignature() -> (UnsafePointer<UInt8>, count: Int)
+
+
+  /// Returns the module that owns this macro.
+  ///
+  /// - Returns: A newly allocated buffer containing the owning module name. The
+  ///   caller is responsible for managing the memory.
+  static func _owningModule() -> (UnsafePointer<UInt8>, count: Int)
+
+  /// Returns the set of modules that are needed (beyond the owning module) to
+  /// process the module signature.
+  ///
+  /// - Returns: A newly allocated buffer containing a string with all of the
+  /// supplemental signature module names, separated by semicolons. The caller
+  /// is responsible for managing the memory.
+  static func _supplementalSignatureModules()
+      -> (UnsafePointer<UInt8>, count: Int)
 }

lib/Sema/CSGen.cpp

@@ -1200,7 +1200,8 @@ namespace {
         if (!protocol)
           return Type();
 
-        auto openedType = CS.getTypeOfMacroReference(kind, expr);
+        auto openedType = CS.getTypeOfMacroReference(
+            ctx.getIdentifier(kind), expr);
         if (!openedType)
           return Type();
 
@@ -3633,7 +3634,7 @@ namespace {
       auto &ctx = CS.getASTContext();
       if (ctx.LangOpts.hasFeature(Feature::Macros)) {
         auto macroIdent = expr->getMacroName().getBaseIdentifier();
-        auto refType = CS.getTypeOfMacroReference(macroIdent.str(), expr);
+        auto refType = CS.getTypeOfMacroReference(macroIdent, expr);
         if (!refType) {
           ctx.Diags.diagnose(expr->getMacroNameLoc(), diag::macro_undefined,
                              macroIdent)