Adjust to the new clang builtin modules

<__stddef_size_t.h> is now a modular header, and so will not be absorbed into Darwin. When that happens, Darwin.C.stddef no longer has a size_t declaration, instead it imports the declaration from _Builtin_stddef.size_t. Darwin still has its other size_t declaration though, Darwin.POSIX._types._size_t, and that's the one that will be found currently. If that one ever goes away though, allow for _Builtin_stddef.size_t being found in C-typedef-Darwin.

Author: ian-twilightcoder

include/swift/ClangImporter/ClangImporter.h

@@ -647,6 +647,7 @@ bool isCxxConstReferenceType(const clang::Type *type);
 struct ClangInvocationFileMapping {
   SmallVector<std::pair<std::string, std::string>, 2> redirectedFiles;
   SmallVector<std::pair<std::string, std::string>, 1> overridenFiles;
+  bool requiresBuiltinHeadersInSystemModules;
 };
 
 /// On Linux, some platform libraries (glibc, libstdc++) are not modularized.

lib/ClangImporter/ClangImporter.cpp

@@ -1204,6 +1204,11 @@ ClangImporter::create(ASTContext &ctx,
   // Create a new Clang compiler invocation.
   {
     importer->Impl.ClangArgs = getClangArguments(ctx);
+    if (fileMapping.requiresBuiltinHeadersInSystemModules) {
+      if (!importerOpts.DirectClangCC1ModuleBuild)
+        importer->Impl.ClangArgs.push_back("-Xclang");
+      importer->Impl.ClangArgs.push_back("-fbuiltin-headers-in-system-modules");
+    }
     ArrayRef<std::string> invocationArgStrs = importer->Impl.ClangArgs;
     if (importerOpts.DumpClangDiagnostics) {
       llvm::errs() << "'";

lib/ClangImporter/ClangIncludePaths.cpp

@@ -398,8 +398,8 @@ static void getLibStdCxxFileMapping(
 }
 
 namespace {
-std::string
-GetWindowsAuxiliaryFile(StringRef modulemap, const SearchPathOptions &Options) {
+std::string GetWindowsAuxiliaryFile(StringRef modulemap,
+                                    const SearchPathOptions &Options) {
   StringRef SDKPath = Options.getSDKPath();
   if (!SDKPath.empty()) {
     llvm::SmallString<261> path{SDKPath};
@@ -420,7 +420,8 @@ GetWindowsAuxiliaryFile(StringRef modulemap, const SearchPathOptions &Options) {
 
 SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     ASTContext &Context,
-    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS) {
+    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS,
+    bool &requiresBuiltinHeadersInSystemModules) {
   const llvm::Triple &Triple = Context.LangOpts.Target;
   const SearchPathOptions &SearchPathOpts = Context.SearchPathOpts;
   SmallVector<std::pair<std::string, std::string>, 2> Mappings;
@@ -468,8 +469,21 @@ SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     llvm::sys::path::append(UCRTInjection, "module.modulemap");
 
     AuxiliaryFile = GetWindowsAuxiliaryFile("ucrt.modulemap", SearchPathOpts);
-    if (!AuxiliaryFile.empty())
+    if (!AuxiliaryFile.empty()) {
+      // The ucrt module map has the C standard library headers all together.
+      // That leads to module cycles with the clang _Builtin_ modules. e.g.
+      // <fenv.h> on ucrt includes <float.h>. The clang builtin <float.h>
+      // include-nexts <float.h>. When both of those UCRT headers are in the
+      // ucrt module, there's a module cycle ucrt -> _Builtin_float -> ucrt
+      // (i.e. fenv.h (ucrt) -> float.h (builtin) -> float.h (ucrt)). Until the
+      // ucrt module map is updated, the builtin headers need to join the system
+      // modules. i.e. when the builtin float.h is in the ucrt module too, the
+      // cycle goes away. Note that -fbuiltin-headers-in-system-modules does
+      // nothing to fix the same problem with C++ headers, and is generally
+      // fragile.
       Mappings.emplace_back(std::string(UCRTInjection), AuxiliaryFile);
+      requiresBuiltinHeadersInSystemModules = true;
+    }
   }
 
   struct {
@@ -515,20 +529,36 @@ ClangInvocationFileMapping swift::getClangInvocationFileMapping(
 
   const llvm::Triple &triple = ctx.LangOpts.Target;
 
+  SmallVector<std::pair<std::string, std::string>, 2> libcFileMapping;
   if (triple.isOSWASI()) {
     // WASI Mappings
-    result.redirectedFiles.append(
-        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs));
+    libcFileMapping =
+        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs);
   } else {
     // Android/BSD/Linux Mappings
-    result.redirectedFiles.append(getLibcFileMapping(
-        ctx, "glibc.modulemap", StringRef("SwiftGlibc.h"), vfs));
+    libcFileMapping = getLibcFileMapping(ctx, "glibc.modulemap",
+                                         StringRef("SwiftGlibc.h"), vfs);
   }
+  result.redirectedFiles.append(libcFileMapping);
+  // Both libc module maps have the C standard library headers all together in a
+  // SwiftLibc module. That leads to module cycles with the clang _Builtin_
+  // modules. e.g. <inttypes.h> includes <stdint.h> on these platforms. The
+  // clang builtin <stdint.h> include-nexts <stdint.h>. When both of those
+  // platform headers are in the SwiftLibc module, there's a module cycle
+  // SwiftLibc -> _Builtin_stdint -> SwiftLibc (i.e. inttypes.h (platform) ->
+  // stdint.h (builtin) -> stdint.h (platform)). Until this can be fixed in
+  // these module maps, the clang builtin headers need to join the "system"
+  // modules (SwiftLibc). i.e. when the clang builtin stdint.h is in the
+  // SwiftLibc module too, the cycle goes away. Note that
+  // -fbuiltin-headers-in-system-modules does nothing to fix the same problem
+  // with C++ headers, and is generally fragile.
+  result.requiresBuiltinHeadersInSystemModules = !libcFileMapping.empty();
 
   if (ctx.LangOpts.EnableCXXInterop)
     getLibStdCxxFileMapping(result, ctx, vfs);
 
-  result.redirectedFiles.append(GetWindowsFileMappings(ctx, vfs));
+  result.redirectedFiles.append(GetWindowsFileMappings(
+      ctx, vfs, result.requiresBuiltinHeadersInSystemModules));
 
   return result;
 }

lib/DriverTool/sil_opt_main.cpp

@@ -518,6 +518,8 @@ struct SILOptOptions {
                       swift::UnavailableDeclOptimization::Complete, "complete",
                       "Eliminate unavailable decls from lowered SIL/IR")),
               llvm::cl::init(swift::UnavailableDeclOptimization::None));
+
+  llvm::cl::list<std::string> ClangXCC = llvm::cl::list<std::string>("Xcc", llvm::cl::desc("option to pass to clang"));
 };
 
 /// Regular expression corresponding to the value given in one of the
@@ -671,6 +673,11 @@ int sil_opt_main(ArrayRef<const char *> argv, void *MainAddr) {
   Invocation.getDiagnosticOptions().VerifyMode =
     options.VerifyMode ? DiagnosticOptions::Verify : DiagnosticOptions::NoVerify;
 
+  auto ExtraArgs = Invocation.getClangImporterOptions().ExtraArgs;
+  for (const auto &xcc : options.ClangXCC) {
+    ExtraArgs.push_back(xcc);
+  }
+
   // Setup the SIL Options.
   SILOptions &SILOpts = Invocation.getSILOptions();
   SILOpts.InlineThreshold = options.SILInlineThreshold;

stdlib/public/Backtracing/SymbolicatedBacktrace.swift

@@ -17,12 +17,14 @@
 
 import Swift
 
-#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS)
-@_implementationOnly import OS.Darwin
-#endif
-
-@_implementationOnly import OS.Libc
+@_implementationOnly import CompressionLibs
+@_implementationOnly import FixedLayout
+@_implementationOnly import ImageFormats
+@_implementationOnly import OS
 @_implementationOnly import Runtime
+@_implementationOnly import _SwiftConcurrencyShims
+@_implementationOnly import SwiftShims
+@_implementationOnly import SwiftOverlayShims
 
 /// A symbolicated backtrace
 public struct SymbolicatedBacktrace: CustomStringConvertible {

test/DebugInfo/C-typedef-Darwin.swift

@@ -9,4 +9,32 @@ let blah = size_t(1024)
 use(blah)
 // CHECK: !DIDerivedType(tag: DW_TAG_typedef,
 // CHECK-SAME:           scope: ![[DARWIN_MODULE:[0-9]+]],
-// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "stddef"
+// size_t is defined in clang, originally by <stddef.h>, and later split out to
+// <__stddef_size_t.h>. clang's original behavior, which is kept up via clang's
+// Darwin toolchain adding -fbuiltin-headers-in-system-modules, makes clang'e
+// <stddef.h> a part of the Darwin.C.stddef module. There are a few different
+// things that can happen depending on the situation.
+// 1. <__stddef_size_t.h> doesn't exist and everything is in clang's <stddef.h>,
+// -fbuiltin-headers-in-system-modules is in use. Darwin.C.stddef precompiles
+// clang's <stddef.h> and has size_t.
+// 2. <__stddef_size_t.h> but doesn't have a module,
+// -fbuiltin-headers-in-system-modules is still in use. This is basically the
+// same as #1 with the added detail that <__stddef_size_t.h> gets absorbed by
+// Darwin.C.stddef
+// 3. <__stddef_size_t.h> does have a module,
+// -fbuiltin-headers-in-system-modules is still in use. Darwin.C.stddef
+// precompiles clang's <stddef.h>, but no longer absorbs <__stddef_size_t.h> and
+// no longer has size_t. Instead _Builtin_stddef.size_t has size_t.
+// 4. -fbuiltin-headers-in-system-modules is no longer used. clang's <stddef.h>
+// is now in _Builtin_stddef which will be imported by Darwin.
+// _Builtin_stddef.size_t still has size_t.
+// There's one more wrinkle: size_t is also defined in macOS by
+// <sys/_types/_size_t.h> currently. That means that Darwin.POSIX._types._size_t
+// also has size_t. That means there are currently two valid modules for size_t.
+// 1. Darwin.C.stddef or _Builtin_stddef.size_t, depending on the state of
+// clang's builtin modules and whether -fbuiltin-headers-in-system-modules is
+// used
+// 2. Darwin.POSIX._types._size_t, as long as <sys/_types/_size_t.h> declares
+// size_t, which is shouldn't do.
+// All all three modules
+// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "{{stddef|size_t|_size_t}}"

test/Interop/lit.local.cfg

@@ -33,9 +33,11 @@ if get_target_os() in ['windows-msvc']:
     # Clang should build object files with link settings equivalent to -libc MD
     # when building for the MSVC target.
     clang_opt = clang_compile_opt + '-D_MT -D_DLL -Xclang --dependent-lib=msvcrt -Xclang --dependent-lib=oldnames '
-    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {}'.format(os.path.join(config.swift_obj_root,
-                                                            'stdlib',
-                                                            'windows-vfs-overlay.yaml'))))
+    # ucrt.modulemap currently requires -fbuiltin-headers-in-system-modules
+    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules'.format(
+                                                            os.path.join(config.swift_obj_root,
+                                                                         'stdlib',
+                                                                         'windows-vfs-overlay.yaml'))))
 else:
     # FIXME(compnerd) do all the targets we currently support use SysV ABI?
     config.substitutions.insert(0, ('%target-abi', 'SYSV'))

test/lit.cfg

@@ -374,10 +374,10 @@ else:
 config.swift_system_overlay_opt = ""
 config.clang_system_overlay_opt = ""
 if kIsWindows:
-    config.swift_system_overlay_opt = "-vfsoverlay {}".format(
+    config.swift_system_overlay_opt = "-vfsoverlay {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
         os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
     )
-    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {}".format(
+    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
         os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
     )
 stdlib_resource_dir_opt = config.resource_dir_opt

utils/build.ps1

@@ -608,7 +608,7 @@ function Build-CMakeProject {
 
           $SwiftArgs += @("-resource-dir", "$SwiftResourceDir")
           $SwiftArgs += @("-L", "$SwiftResourceDir\windows")
-          $SwiftArgs += @("-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml")
+          $SwiftArgs += @("-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml", "-Xcc", "-Xclang", "-Xcc", "-fbuiltin-headers-in-system-modules")
         }
       } else {
         $SwiftArgs += @("-sdk", "$BinaryCache\toolchains\$PinnedToolchain\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk")