Merge pull request #8570 from apple/eng/blanmuir/finclude-tree-preserve-pch-path-stable

[cas] Add cc1 option -finclude-tree-preserve-pch-path

Author: benlangmuir

clang/include/clang/CAS/IncludeTree.h

@@ -794,12 +794,12 @@ class IncludeTreeRoot : public IncludeTreeBase<IncludeTreeRoot> {
     return IncludeTree::FileList(std::move(*Node));
   }
 
-  Expected<std::optional<StringRef>> getPCHBuffer() {
+  Expected<std::optional<IncludeTree::File>> getPCH() {
     if (std::optional<ObjectRef> Ref = getPCHRef()) {
       auto Node = getCAS().getProxy(*Ref);
       if (!Node)
         return Node.takeError();
-      return Node->getData();
+      return IncludeTree::File(std::move(*Node));
     }
     return std::nullopt;
   }

clang/include/clang/Driver/Options.td

@@ -7119,6 +7119,11 @@ defm casid_output : BoolFOption<"casid-output",
                          " write a CASID for the output file.">,
     NegFlag<SetFalse>>;
 
+defm include_tree_preserve_pch_path : BoolFOption<"include-tree-preserve-pch-path",
+    FrontendOpts<"IncludeTreePreservePCHPath">, DefaultFalse,
+    PosFlag<SetTrue, [], "Keep the original PCH path in include-tree rather than canonicalizing">,
+    NegFlag<SetFalse>>;
+
 /// BEGIN MCCAS
 defm cas_backend : BoolFOption<"cas-backend",
     CodeGenOpts<"UseCASBackend">, DefaultFalse,

clang/include/clang/Frontend/FrontendOptions.h

@@ -372,6 +372,11 @@ class FrontendOptions {
   /// caching of compilation outputs. This is used for testing purposes.
   unsigned DisableCachedCompileJobReplay : 1;
 
+  /// Whether to preserve the original PCH path in the include-tree, or to
+  /// canonicalize it to a fixed value. Setting this to \c true allows the use
+  /// of gmodules with PCH and include tree.
+  unsigned IncludeTreePreservePCHPath : 1;
+
   /// Keep the diagnostic client open for receiving diagnostics after the source
   /// files have been processed.
   unsigned MayEmitDiagnosticsAfterProcessingSourceFiles : 1;
@@ -593,6 +598,7 @@ class FrontendOptions {
         BuildingImplicitModuleUsesLock(true), ModulesEmbedAllFiles(false),
         IncludeTimestamps(true), UseTemporary(true), CacheCompileJob(false),
         ForIncludeTreeScan(false), DisableCachedCompileJobReplay(false),
+        IncludeTreePreservePCHPath(false),
         MayEmitDiagnosticsAfterProcessingSourceFiles(false),
         WriteOutputAsCASID(false), AllowPCMWithCompilerErrors(false),
         ModulesShareFileManager(true), EmitSymbolGraph(false),

clang/lib/CAS/IncludeTree.cpp

@@ -832,10 +832,13 @@ llvm::Error IncludeTree::APINotes::forEachAPINotes(
 }
 
 llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) {
-  if (std::optional<ObjectRef> PCHRef = getPCHRef()) {
+  std::optional<IncludeTree::File> PCH;
+  if (llvm::Error E = getPCH().moveInto(PCH))
+    return E;
+  if (PCH) {
     OS.indent(Indent) << "(PCH) ";
-    getCAS().getID(*PCHRef).print(OS);
-    OS << '\n';
+    if (llvm::Error E = PCH->print(OS))
+      return E;
   }
   std::optional<cas::IncludeTree> MainTree;
   if (llvm::Error E = getMainFileTree().moveInto(MainTree))

clang/lib/Frontend/FrontendAction.cpp

@@ -998,7 +998,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   };
 
   std::optional<cas::IncludeTreeRoot> IncludeTreeRoot;
-  std::optional<StringRef> IncludeTreePCHBuffer;
+  std::optional<cas::IncludeTree::File> IncludeTreePCH;
   if (Input.isIncludeTree()) {
     if (llvm::Error E = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(),
                                                   Input.getIncludeTree())
@@ -1012,8 +1012,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     CI.getPreprocessor().setPPCachedActions(std::move(*PPCachedAct));
     CI.getFrontendOpts().IncludeTimestamps = false;
 
-    if (llvm::Error E =
-            IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer))
+    if (llvm::Error E = IncludeTreeRoot->getPCH().moveInto(IncludeTreePCH))
       return reportError(std::move(E));
 
     auto ModMap = IncludeTreeRoot->getModuleMap();
@@ -1148,7 +1147,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
       CI.getASTContext().setExternalSource(source);
     } else if (CI.getLangOpts().Modules ||
                !CI.getPreprocessorOpts().ImplicitPCHInclude.empty() ||
-               IncludeTreePCHBuffer) {
+               IncludeTreePCH) {
       // Use PCM or PCH.
       assert(hasPCHSupport() && "This action does not have PCH support!");
       ASTDeserializationListener *DeserialListener =
@@ -1167,16 +1166,17 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
         DeleteDeserialListener = true;
       }
       if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty() ||
-          IncludeTreePCHBuffer) {
+          IncludeTreePCH) {
         StringRef PCHPath;
         DisableValidationForModuleKind DisableValidation;
         std::unique_ptr<llvm::MemoryBuffer> PCHBuffer;
-        if (IncludeTreePCHBuffer) {
-          PCHPath = "<PCH>";
+        if (IncludeTreePCH) {
           // No need to do any validation.
           DisableValidation = DisableValidationForModuleKind::All;
-          PCHBuffer =
-              llvm::MemoryBuffer::getMemBuffer(*IncludeTreePCHBuffer, PCHPath);
+          if (llvm::Error E =
+                  IncludeTreePCH->getMemoryBuffer().moveInto(PCHBuffer))
+            return reportError(std::move(E));
+          PCHPath = PCHBuffer->getBufferIdentifier();
         } else {
           PCHPath = CI.getPreprocessorOpts().ImplicitPCHInclude;
           DisableValidation =

clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp

@@ -644,9 +644,17 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance,
         FM.getObjectRefForFileContent(PPOpts.ImplicitPCHInclude);
     if (!CASContents)
       return llvm::errorCodeToError(CASContents.getError());
-    PCHRef = **CASContents;
 
-    return Error::success();
+    StringRef PCHFilename = "<PCH>";
+    if (NewInvocation.getFrontendOpts().IncludeTreePreservePCHPath)
+      PCHFilename = PPOpts.ImplicitPCHInclude;
+
+    auto PCHFile =
+        cas::IncludeTree::File::create(DB, PCHFilename, **CASContents);
+    if (!PCHFile)
+      return PCHFile.takeError();
+    PCHRef = PCHFile->getRef();
+    return llvm::Error::success();
   };
 
   if (Error E = FinishIncludeTree())

clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp

@@ -58,6 +58,9 @@ void tooling::dependencies::configureInvocationForCaching(
     HSOpts.ResourceDir = std::move(OriginalHSOpts.ResourceDir);
     // Preserve fmodule-file options.
     HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles);
+    // Preserve -gmodules (see below for caveats).
+    HSOpts.ModuleFormat = OriginalHSOpts.ModuleFormat;
+
     auto &PPOpts = CI.getPreprocessorOpts();
     // We don't need this because we save the contents of the PCH file in the
     // include tree root.
@@ -72,12 +75,13 @@ void tooling::dependencies::configureInvocationForCaching(
       PPOpts.MacroIncludes.clear();
       PPOpts.Includes.clear();
     }
-    // Disable `-gmodules` to avoid debug info referencing a non-existent PCH
-    // filename.
-    // NOTE: We'd have to preserve \p HeaderSearchOptions::ModuleFormat (code
-    // above resets \p HeaderSearchOptions) when properly supporting
-    // `-gmodules`.
-    CI.getCodeGenOpts().DebugTypeExtRefs = false;
+    if (!FrontendOpts.IncludeTreePreservePCHPath) {
+      // Disable `-gmodules` to avoid debug info referencing a non-existent PCH
+      // filename.
+      // FIXME: we should also allow -gmodules if there is no PCH involved.
+      CI.getCodeGenOpts().DebugTypeExtRefs = false;
+      HSOpts.ModuleFormat = "raw";
+    }
     // Clear APINotes options.
     CI.getAPINotesOpts().ModuleSearchPaths = {};
   } else {

clang/test/ClangScanDeps/include-tree-preserve-pch-path.c

@@ -0,0 +1,63 @@
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb_no_preserve.json.template > %t/cdb_no_preserve.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_pch.json
+// RUN: FileCheck %s -input-file %t/deps_pch.json -DPREFIX=%/t
+
+// CHECK: "-fmodule-format=obj"
+// CHECK: "-dwarf-ext-refs"
+
+// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp
+// RUN: %clang @%t/pch.rsp
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_tu.json
+// RUN: FileCheck %s -input-file %t/deps_tu.json -DPREFIX=%/t
+
+// RUN: %deps-to-rsp %t/deps_tu.json --tu-index 0 > %t/tu.rsp
+// RUN: %clang @%t/tu.rsp
+
+// RUN: cat %t/tu.ll | FileCheck %s -check-prefix=LLVMIR -DPREFIX=%/t
+// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "prefix.pch"
+
+// Extract include-tree casid
+// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid
+
+// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -check-prefix=INCLUDE_TREE -DPREFIX=%/t
+// INCLUDE_TREE: (PCH) [[PREFIX]]/prefix.pch llvmcas://
+
+// RUN: clang-scan-deps -compilation-database %t/cdb_no_preserve.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_no_preserve.json
+// RUN: FileCheck %s -input-file %t/deps_no_preserve.json -DPREFIX=%/t -check-prefix=NO_PRESERVE
+
+// Note: "raw" is the default format, so it will not show up in the arguments.
+// NO_PRESERVE-NOT: "-fmodule-format=
+// NO_PRESERVE-NOT: "-dwarf-ext-refs"
+
+
+//--- cdb_pch.json.template
+[{
+  "directory": "DIR",
+  "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g -Xclang -finclude-tree-preserve-pch-path",
+  "file": "DIR/prefix.h"
+}]
+
+//--- cdb.json.template
+[{
+  "directory": "DIR",
+  "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g -Xclang -finclude-tree-preserve-pch-path",
+  "file": "DIR/tu.c"
+}]
+
+//--- cdb_no_preserve.json.template
+[{
+  "directory": "DIR",
+  "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g",
+  "file": "DIR/tu.c"
+}]
+
+//--- prefix.h
+struct S {};
+
+//--- tu.c
+struct S s;

clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c

@@ -47,7 +47,7 @@
 // CHECK-NOT: Bar
 
 // CHECK-LABEL: TRANSLATION UNIT
-// CHECK: (PCH) llvmcas://
+// CHECK: (PCH) <PCH> llvmcas://
 // CHECK: [[PREFIX]]/tu.c llvmcas://
 // CHECK: 1:1 <built-in> llvmcas://
 // CHECK: 2:1 (Spurious import) (Module) Foo.Bar [[PREFIX]]/Foo.framework/Headers/Bar.h llvmcas://

clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c

@@ -53,7 +53,7 @@
 // CHECK:   3:1 (Module) Mod_Private
 
 // CHECK-LABEL: TRANSLATION UNIT
-// CHECK: (PCH) llvmcas://
+// CHECK: (PCH) <PCH> llvmcas://
 // CHECK: [[PREFIX]]/tu.m llvmcas://
 // CHECK: 1:1 <built-in> llvmcas://
 // CHECK: 2:1 (Module) Mod_Private