[ClangImporter] Swift needs to pass -Xclang -fbuiltin-headers-in-system-modules for its module maps that group cstd headers

Swift has some module maps it overlays on Linux and Windows that groups all of the C standard library headers into a single module. This doesn’t allow clang and C++ headers to layer properly with the OS/SDK modules. clang will set -fbuiltin-headers-in-system-modules as necessary for Apple SDKs, but Swift will need to pass that flag itself when required by its module maps.

Author: ian-twilightcoder

include/swift/ClangImporter/ClangImporter.h

@@ -647,6 +647,7 @@ bool isCxxConstReferenceType(const clang::Type *type);
 struct ClangInvocationFileMapping {
   SmallVector<std::pair<std::string, std::string>, 2> redirectedFiles;
   SmallVector<std::pair<std::string, std::string>, 1> overridenFiles;
+  bool requiresBuiltinHeadersInSystemModules;
 };
 
 /// On Linux, some platform libraries (glibc, libstdc++) are not modularized.

lib/ClangImporter/ClangImporter.cpp

@@ -1204,6 +1204,11 @@ ClangImporter::create(ASTContext &ctx,
   // Create a new Clang compiler invocation.
   {
     importer->Impl.ClangArgs = getClangArguments(ctx);
+    if (fileMapping.requiresBuiltinHeadersInSystemModules) {
+      if (!importerOpts.DirectClangCC1ModuleBuild)
+        importer->Impl.ClangArgs.push_back("-Xclang");
+      importer->Impl.ClangArgs.push_back("-fbuiltin-headers-in-system-modules");
+    }
     ArrayRef<std::string> invocationArgStrs = importer->Impl.ClangArgs;
     if (importerOpts.DumpClangDiagnostics) {
       llvm::errs() << "'";

lib/ClangImporter/ClangIncludePaths.cpp

@@ -420,7 +420,8 @@ GetWindowsAuxiliaryFile(StringRef modulemap, const SearchPathOptions &Options) {
 
 SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     ASTContext &Context,
-    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS) {
+    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS,
+    bool &requiresBuiltinHeadersInSystemModules) {
   const llvm::Triple &Triple = Context.LangOpts.Target;
   const SearchPathOptions &SearchPathOpts = Context.SearchPathOpts;
   SmallVector<std::pair<std::string, std::string>, 2> Mappings;
@@ -468,8 +469,21 @@ SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     llvm::sys::path::append(UCRTInjection, "module.modulemap");
 
     AuxiliaryFile = GetWindowsAuxiliaryFile("ucrt.modulemap", SearchPathOpts);
-    if (!AuxiliaryFile.empty())
+    if (!AuxiliaryFile.empty()) {
+      // The ucrt module map has the C standard library headers all together.
+      // That leads to module cycles with the clang _Builtin_ modules. e.g.
+      // <fenv.h> on ucrt includes <float.h>. The clang builtin <float.h>
+      // include-nexts <float.h>. When both of those UCRT headers are in the
+      // ucrt module, there's a module cycle ucrt -> _Builtin_float -> ucrt
+      // (i.e. fenv.h (ucrt) -> float.h (builtin) -> float.h (ucrt)). Until the
+      // ucrt module map is updated, the builtin headers need to join the system
+      // modules. i.e. when the builtin float.h is in the ucrt module too, the
+      // cycle goes away. Note that -fbuiltin-headers-in-system-modules does
+      // nothing to fix the same problem with C++ headers, and is generally
+      // fragile.
       Mappings.emplace_back(std::string(UCRTInjection), AuxiliaryFile);
+      requiresBuiltinHeadersInSystemModules = true;
+    }
   }
 
   struct {
@@ -515,20 +529,36 @@ ClangInvocationFileMapping swift::getClangInvocationFileMapping(
 
   const llvm::Triple &triple = ctx.LangOpts.Target;
 
+  SmallVector<std::pair<std::string, std::string>, 2> libcFileMapping;
   if (triple.isOSWASI()) {
     // WASI Mappings
-    result.redirectedFiles.append(
-        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs));
+    libcFileMapping =
+        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs);
   } else {
     // Android/BSD/Linux Mappings
-    result.redirectedFiles.append(getLibcFileMapping(
-        ctx, "glibc.modulemap", StringRef("SwiftGlibc.h"), vfs));
+    libcFileMapping = getLibcFileMapping(ctx, "glibc.modulemap",
+                                         StringRef("SwiftGlibc.h"), vfs);
   }
+  result.redirectedFiles.append(libcFileMapping);
+  // Both libc module maps have the C standard library headers all together in a
+  // SwiftLibc module. That leads to module cycles with the clang _Builtin_
+  // modules. e.g. <inttypes.h> includes <stdint.h> on these platforms. The
+  // clang builtin <stdint.h> include-nexts <stdint.h>. When both of those
+  // platform headers are in the SwiftLibc module, there's a module cycle
+  // SwiftLibc -> _Builtin_stdint -> SwiftLibc (i.e. inttypes.h (platform) ->
+  // stdint.h (builtin) -> stdint.h (platform)). Until this can be fixed in
+  // these module maps, the clang builtin headers need to join the "system"
+  // modules (SwiftLibc). i.e. when the clang builtin stdint.h is in the
+  // SwiftLibc module too, the cycle goes away. Note that
+  // -fbuiltin-headers-in-system-modules does nothing to fix the same problem
+  // with C++ headers, and is generally fragile.
+  result.requiresBuiltinHeadersInSystemModules = !libcFileMapping.empty();
 
   if (ctx.LangOpts.EnableCXXInterop)
     getLibStdCxxFileMapping(result, ctx, vfs);
 
-  result.redirectedFiles.append(GetWindowsFileMappings(ctx, vfs));
+  result.redirectedFiles.append(GetWindowsFileMappings(
+      ctx, vfs, result.requiresBuiltinHeadersInSystemModules));
 
   return result;
 }

lib/DriverTool/sil_opt_main.cpp

@@ -518,6 +518,8 @@ struct SILOptOptions {
                       swift::UnavailableDeclOptimization::Complete, "complete",
                       "Eliminate unavailable decls from lowered SIL/IR")),
               llvm::cl::init(swift::UnavailableDeclOptimization::None));
+
+  llvm::cl::list<std::string> ClangXCC = llvm::cl::list<std::string>("Xcc", llvm::cl::desc("option to pass to clang"));
 };
 
 /// Regular expression corresponding to the value given in one of the
@@ -671,6 +673,11 @@ int sil_opt_main(ArrayRef<const char *> argv, void *MainAddr) {
   Invocation.getDiagnosticOptions().VerifyMode =
     options.VerifyMode ? DiagnosticOptions::Verify : DiagnosticOptions::NoVerify;
 
+  ClangImporterOptions &clangImporterOptions = Invocation.getClangImporterOptions();
+  for (const auto &xcc : options.ClangXCC) {
+      clangImporterOptions.ExtraArgs.push_back(xcc);
+  }
+
   // Setup the SIL Options.
   SILOptions &SILOpts = Invocation.getSILOptions();
   SILOpts.InlineThreshold = options.SILInlineThreshold;

stdlib/cmake/modules/AddSwiftStdlib.cmake

@@ -929,7 +929,7 @@ function(add_swift_target_library_single target name)
         -Xcc;-Xclang;-Xcc;-ivfsoverlay;-Xcc;-Xclang;-Xcc;${SWIFTLIB_SINGLE_VFS_OVERLAY})
     endif()
     list(APPEND SWIFTLIB_SINGLE_SWIFT_COMPILE_FLAGS
-      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}")
+      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}";-Xcc;-Xclang;-Xcc;-fbuiltin-headers-in-system-modules)
     if(NOT CMAKE_HOST_SYSTEM MATCHES Windows)
       swift_windows_include_for_arch(${SWIFTLIB_SINGLE_ARCHITECTURE} SWIFTLIB_INCLUDE)
       foreach(directory ${SWIFTLIB_INCLUDE})
@@ -2659,7 +2659,7 @@ function(_add_swift_target_executable_single name)
 
   if(SWIFTEXE_SINGLE_SDK STREQUAL "WINDOWS")
     list(APPEND SWIFTEXE_SINGLE_COMPILE_FLAGS
-      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}")
+      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}";-Xcc;-Xclang;-Xcc;-fbuiltin-headers-in-system-modules)
   endif()
 
   if ("${SWIFTEXE_SINGLE_SDK}" STREQUAL "LINUX")

stdlib/public/Backtracing/SymbolicatedBacktrace.swift

@@ -23,6 +23,11 @@ import Swift
 
 @_implementationOnly import OS.Libc
 @_implementationOnly import Runtime
+// Because we've turned off the OS/SDK modules, and we don't have a module for
+// stddef.h, and we sometimes build with -fbuiltin-headers-in-system-modules for
+// vfs reasons, stddef.h can be absorbed into a random module. Sometimes it's
+// SwiftOverlayShims.
+@_implementationOnly import SwiftOverlayShims
 
 /// A symbolicated backtrace
 public struct SymbolicatedBacktrace: CustomStringConvertible {

test/DebugInfo/C-typedef-Darwin.swift

@@ -9,4 +9,13 @@ let blah = size_t(1024)
 use(blah)
 // CHECK: !DIDerivedType(tag: DW_TAG_typedef,
 // CHECK-SAME:           scope: ![[DARWIN_MODULE:[0-9]+]],
-// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "stddef"
+
+// size_t is defined in clang, originally by <stddef.h>, and later split out to
+// <__stddef_size_t.h>. Depending on the state of clang's builtin headers and
+// modules, size_t can be in either Darwin.C.stddef or _Builtin_stddef.size_t.
+// size_t is also defined in macOS by <sys/_types/_size_t.h> which is in the
+// Darwin.POSIX._types._size_t module. Ideally Apple will remove the duplicate
+// declaration and clang will settle to _Builtin_stddef.size_t, but while it's
+// all in flux allow all three of those modules.
+// Darwin.C.stddef|_Builtin_stddef.size_t|Darwin.POSIX._types._size_t
+// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "{{stddef|size_t|_size_t}}"

test/Interop/lit.local.cfg

@@ -33,9 +33,11 @@ if get_target_os() in ['windows-msvc']:
     # Clang should build object files with link settings equivalent to -libc MD
     # when building for the MSVC target.
     clang_opt = clang_compile_opt + '-D_MT -D_DLL -Xclang --dependent-lib=msvcrt -Xclang --dependent-lib=oldnames '
-    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {}'.format(os.path.join(config.swift_obj_root,
-                                                            'stdlib',
-                                                            'windows-vfs-overlay.yaml'))))
+    # ucrt.modulemap currently requires -fbuiltin-headers-in-system-modules
+    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules'.format(
+                                                            os.path.join(config.swift_obj_root,
+                                                                         'stdlib',
+                                                                         'windows-vfs-overlay.yaml'))))
 else:
     # FIXME(compnerd) do all the targets we currently support use SysV ABI?
     config.substitutions.insert(0, ('%target-abi', 'SYSV'))

test/lit.cfg

@@ -374,10 +374,10 @@ else:
 config.swift_system_overlay_opt = ""
 config.clang_system_overlay_opt = ""
 if kIsWindows:
-    config.swift_system_overlay_opt = "-vfsoverlay {}".format(
+    config.swift_system_overlay_opt = "-vfsoverlay {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
         os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
     )
-    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {}".format(
+    config.clang_system_overlay_opt = "-Xcc -ivfsoverlay -Xcc {} -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules".format(
         os.path.join(config.swift_obj_root, "stdlib", "windows-vfs-overlay.yaml")
     )
 stdlib_resource_dir_opt = config.resource_dir_opt
@@ -1592,7 +1592,9 @@ elif run_os in ['windows-msvc']:
     config.target_swift_modulewrap =                                             \
             ('%r -modulewrap -target %s' % (config.swiftc, config.variant_triple))
     config.target_swift_emit_pcm =                                               \
-            ('%r -emit-pcm -target %s' % (config.swiftc, config.variant_triple))
+            ('%r -emit-pcm -target %s %s' % (config.swiftc,                      \
+                                             config.variant_triple,              \
+                                             config.swift_system_overlay_opt))
 
 
 elif (run_os in ['linux-gnu', 'linux-gnueabihf', 'freebsd', 'openbsd', 'windows-cygnus', 'windows-gnu'] or

utils/build.ps1

@@ -608,7 +608,7 @@ function Build-CMakeProject {
 
           $SwiftArgs += @("-resource-dir", "$SwiftResourceDir")
           $SwiftArgs += @("-L", "$SwiftResourceDir\windows")
-          $SwiftArgs += @("-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml")
+          $SwiftArgs += @("-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml", "-Xcc", "-Xclang", "-Xcc", "-fbuiltin-headers-in-system-modules")
         }
       } else {
         $SwiftArgs += @("-sdk", "$BinaryCache\toolchains\$PinnedToolchain\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk")