WIP: Serialize pending constexr instantiations

I wanted to avoid loading eagerly all function templates
and their instantiations -
even if ultimately `PendingInstantiations` - which contains the same information
is going to be loaded at the end of the TU.

So instead, when we find the definition of a constexpr
function template we load the associated instantiation from the
external modules at that point.

I'm not sure there is a better way to get to a DeclID from a Decl*
than to iterate over all redeclarations?

I still need to cleanup but I'd like feedback on direction first!

Author: cor3ntin

clang/include/clang/Sema/ExternalSemaSource.h

@@ -181,6 +181,9 @@ class ExternalSemaSource : public ExternalASTSource {
                  SmallVectorImpl<std::pair<ValueDecl *,
                                            SourceLocation> > &Pending) {}
 
+  virtual void ReadPendingOfInstantiationsForConstexprEntity(
+      const NamedDecl *D, llvm::SmallSetVector<NamedDecl *, 4> &Decls){};
+
   /// Read the set of late parsed template functions for this source.
   ///
   /// The external source should insert its own late parsed template functions

clang/include/clang/Sema/MultiplexExternalSemaSource.h

@@ -319,6 +319,9 @@ class MultiplexExternalSemaSource : public ExternalSemaSource {
   void ReadPendingInstantiations(
      SmallVectorImpl<std::pair<ValueDecl*, SourceLocation> >& Pending) override;
 
+  virtual void ReadPendingOfInstantiationsForConstexprEntity(
+      const NamedDecl *D, llvm::SmallSetVector<NamedDecl *, 4> &Decls) override;
+
   /// Read the set of late parsed template functions for this source.
   ///
   /// The external source should insert its own late parsed template functions

clang/include/clang/Serialization/ASTBitCodes.h

@@ -695,6 +695,10 @@ enum ASTRecordTypes {
   /// Record code for an unterminated \#pragma clang assume_nonnull begin
   /// recorded in a preamble.
   PP_ASSUME_NONNULL_LOC = 67,
+
+  /// Record code for constexpr templated entities that have been used but not
+  /// yet instantiated.
+  PENDING_INSTANTIATIONS_OF_CONSTEXPR_ENTITIES = 68,
 };
 
 /// Record types used within a source manager block.

clang/include/clang/Serialization/ASTReader.h

@@ -814,6 +814,9 @@ class ASTReader
   /// is the instantiation location.
   SmallVector<serialization::DeclID, 64> PendingInstantiations;
 
+  llvm::DenseMap<serialization::DeclID, std::set<serialization::DeclID>>
+      PendingInstantiationsOfConstexprEntities;
+
   //@}
 
   /// \name DiagnosticsEngine-relevant special data
@@ -2101,6 +2104,9 @@ class ASTReader
                   SmallVectorImpl<std::pair<ValueDecl *,
                                             SourceLocation>> &Pending) override;
 
+  virtual void ReadPendingOfInstantiationsForConstexprEntity(
+      const NamedDecl *D, llvm::SmallSetVector<NamedDecl *, 4> &Decls) override;
+
   void ReadLateParsedTemplates(
       llvm::MapVector<const FunctionDecl *, std::unique_ptr<LateParsedTemplate>>
           &LPTMap) override;

clang/lib/Sema/MultiplexExternalSemaSource.cpp

@@ -310,6 +310,12 @@ void MultiplexExternalSemaSource::ReadPendingInstantiations(
     Sources[i]->ReadPendingInstantiations(Pending);
 }
 
+void MultiplexExternalSemaSource::ReadPendingOfInstantiationsForConstexprEntity(
+    const NamedDecl *D, llvm::SmallSetVector<NamedDecl *, 4> &Decls) {
+  for (size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadPendingOfInstantiationsForConstexprEntity(D, Decls);
+};
+
 void MultiplexExternalSemaSource::ReadLateParsedTemplates(
     llvm::MapVector<const FunctionDecl *, std::unique_ptr<LateParsedTemplate>>
         &LPTMap) {

clang/lib/Sema/SemaTemplateInstantiateDecl.cpp

@@ -6501,15 +6501,25 @@ void Sema::PerformPendingInstantiations(bool LocalOnly) {
 // see CWG2497.
 void Sema::InstantiateFunctionTemplateSpecializations(
     SourceLocation PointOfInstantiation, FunctionDecl *Tpl) {
+
+  auto InstantiateAll = [&](const auto &Range) {
+    for (NamedDecl *Fun : Range) {
+      InstantiateFunctionDefinition(PointOfInstantiation,
+                                    cast<FunctionDecl>(Fun));
+    }
+  };
+
   auto It =
       PendingInstantiationsOfConstexprEntities.find(Tpl->getCanonicalDecl());
-  if (It == PendingInstantiationsOfConstexprEntities.end())
-    return;
-  for (NamedDecl *Fun : It->second) {
-    InstantiateFunctionDefinition(PointOfInstantiation,
-                                  cast<FunctionDecl>(Fun));
+  if (It != PendingInstantiationsOfConstexprEntities.end()) {
+    InstantiateAll(It->second);
+  }
+
+  llvm::SmallSetVector<NamedDecl *, 4> Decls;
+  if (ExternalSource) {
+    ExternalSource->ReadPendingOfInstantiationsForConstexprEntity(Tpl, Decls);
+    InstantiateAll(Decls);
   }
-  PendingInstantiationsOfConstexprEntities.erase(It);
 }
 
 void Sema::PerformDependentDiagnostics(const DeclContext *Pattern,

clang/lib/Serialization/ASTReader.cpp

@@ -3709,6 +3709,19 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       }
       break;
 
+    case PENDING_INSTANTIATIONS_OF_CONSTEXPR_ENTITIES:
+      if (Record.size() % 2 != 0)
+        return llvm::createStringError(
+            std::errc::illegal_byte_sequence,
+            "Invalid PENDING_INSTANTIATIONS_OF_CONSTEXPR_ENTITIES block");
+
+      for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
+        DeclID Key = getGlobalDeclID(F, Record[I++]);
+        DeclID Value = getGlobalDeclID(F, Record[I++]);
+        PendingInstantiationsOfConstexprEntities[Key].insert(Value);
+      }
+      break;
+
     case SEMA_DECL_REFS:
       if (Record.size() != 3)
         return llvm::createStringError(std::errc::illegal_byte_sequence,
@@ -8718,6 +8731,18 @@ void ASTReader::ReadPendingInstantiations(
   PendingInstantiations.clear();
 }
 
+void ASTReader::ReadPendingOfInstantiationsForConstexprEntity(
+    const NamedDecl *D, llvm::SmallSetVector<NamedDecl *, 4> &Decls) {
+  for (auto *Redecl : D->redecls()) {
+    DeclID Id = Redecl->getGlobalID();
+    auto It = PendingInstantiationsOfConstexprEntities.find(Id);
+    if (It == PendingInstantiationsOfConstexprEntities.end())
+      continue;
+    for (DeclID InstantiationId : It->second)
+      Decls.insert(cast<NamedDecl>(GetDecl(InstantiationId)));
+  }
+}
+
 void ASTReader::ReadLateParsedTemplates(
     llvm::MapVector<const FunctionDecl *, std::unique_ptr<LateParsedTemplate>>
         &LPTMap) {

clang/lib/Serialization/ASTWriter.cpp

@@ -849,6 +849,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(SEMA_DECL_REFS);
   RECORD(WEAK_UNDECLARED_IDENTIFIERS);
   RECORD(PENDING_IMPLICIT_INSTANTIATIONS);
+  RECORD(PENDING_INSTANTIATIONS_OF_CONSTEXPR_ENTITIES);
   RECORD(UPDATE_VISIBLE);
   RECORD(DECL_UPDATE_OFFSETS);
   RECORD(DECL_UPDATES);
@@ -4836,6 +4837,16 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   assert(SemaRef.PendingLocalImplicitInstantiations.empty() &&
          "There are local ones at end of translation unit!");
 
+  // Build a record containing all of pending instantiations of constexpr
+  // entities.
+  RecordData PendingInstantiationsOfConstexprEntities;
+  for (const auto &I : SemaRef.PendingInstantiationsOfConstexprEntities) {
+    for (const auto &Elem : I.second) {
+      AddDeclRef(I.first, PendingInstantiationsOfConstexprEntities);
+      AddDeclRef(Elem, PendingInstantiationsOfConstexprEntities);
+    }
+  }
+
   // Build a record containing some declaration references.
   RecordData SemaDeclRefs;
   if (SemaRef.StdNamespace || SemaRef.StdBadAlloc || SemaRef.StdAlignValT) {
@@ -5153,6 +5164,11 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   if (!PendingInstantiations.empty())
     Stream.EmitRecord(PENDING_IMPLICIT_INSTANTIATIONS, PendingInstantiations);
 
+  // Write the record containing pending instantiations of constexpr entities.
+  if (!PendingInstantiationsOfConstexprEntities.empty())
+    Stream.EmitRecord(PENDING_INSTANTIATIONS_OF_CONSTEXPR_ENTITIES,
+                      PendingInstantiationsOfConstexprEntities);
+
   // Write the record containing declaration references of Sema.
   if (!SemaDeclRefs.empty())
     Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);

clang/test/PCH/instantiate-used-constexpr-function.cpp

@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -std=c++2a -emit-pch %s -o %t
+// RUN: %clang_cc1 -std=c++2a -include-pch %t -verify %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+template<typename T> constexpr T f();
+constexpr int g() { return f<int>(); } // #1
+
+#else /*included pch*/
+
+template<typename T> constexpr T f() { return 123; }
+int k[g()];
+
+#endif // HEADER