swiftlang · bnbarham · Aug 30, 2023 · Aug 19, 2023 · Aug 24, 2023 · Aug 24, 2023
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -794,6 +794,7 @@ include(SwiftConfigureSDK)
 include(SwiftComponents)
 include(SwiftList)
 include(AddPureSwift)
+include(SetRPATH)
 
 # Configure swift include, install, build components.
 swift_configure_components()
@@ -952,6 +953,12 @@ if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR)
   else()
     set(SWIFT_SWIFT_PARSER TRUE)
     include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS})
+
+    if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX" AND NOT BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS")
+      # Only "HOSTTOOLS" is supported in Linux when Swift parser integration is enabled.
+      message(WARNING "Force setting BOOTSTRAPPING=HOSTTOOLS because Swift parser integration is enabled")
+      set(BOOTSTRAPPING_MODE "HOSTTOOLS")
+    endif()
   endif()
 endif()
 

@@ -235,7 +235,7 @@ else()
   #
   # step 1: generate a dummy source file, which just includes all headers
   # defined in include/swift/module.modulemap
-  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp"
        "
 #include \"Basic/BridgedSwiftObject.h\"
 #include \"Basic/BasicBridging.h\"
@@ -251,6 +251,12 @@ else()
 
 #include \"Parse/RegexParserBridging.h\"
 ")
+  add_custom_command(
+    OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp"
+      "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+  )
 
   # step 2: build a library containing that source file. This library depends on all the included header files.
   #         The swift modules can now depend on that target.

diff --git a/cmake/modules/AddPureSwift.cmake b/cmake/modules/AddPureSwift.cmake
@@ -65,6 +65,25 @@ function(_add_host_swift_compile_options name)
   _add_host_variant_swift_sanitizer_flags(${name})
 endfunction()
 
+function(_set_pure_swift_link_flags name relpath_to_lib_dir)
+  if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+    # Don't add builder's stdlib RPATH automatically.
+    target_compile_options(${name} PRIVATE
+      $<$<COMPILE_LANGUAGE:Swift>:-no-toolchain-stdlib-rpath>
+    )
+
+    set_property(TARGET ${name}
+      APPEND PROPERTY INSTALL_RPATH
+        # At runtime, use swiftCore in the current just-built toolchain.
+        # NOTE: This relies on the ABI being the same as the builder.
+        "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}"
+    )
+    # NOTE: At this point we don't have any pure swift executables/shared
+    # libraries required for building runtime/stdlib. So we don't need to add
+    # RPATH to the builder's runtime.
+  endif()
+endfunction()
+
 # Add a new "pure" Swift host library.
 #
 # "Pure" Swift host libraries can only contain Swift code, and will be built
@@ -266,6 +285,7 @@ function(add_pure_swift_host_tool name)
   # Create the library.
   add_executable(${name} ${APSHT_SOURCES})
   _add_host_swift_compile_options(${name})
+  _set_pure_swift_link_flags(${name} "../lib")
 
   if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS)
     set_property(TARGET ${name}

diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake
@@ -548,11 +548,11 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
       target_link_libraries(${target} PRIVATE "swiftCore")
 
       target_link_directories(${target} PRIVATE ${host_lib_dir})
-      if(ASRLF_BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS")
-        set(swift_runtime_rpath "${host_lib_dir}")
-      else()
-        set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
-      endif()
+
+      # At runtime, use swiftCore in the current toolchain.
+      # For building stdlib, LD_LIBRARY_PATH will be set to builder's stdlib
+      # FIXME: This assumes the ABI hasn't changed since the builder.
+      set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
 
     elseif(ASRLF_BOOTSTRAPPING_MODE STREQUAL "BOOTSTRAPPING")
       # At build time link against the built swift libraries from the
@@ -576,9 +576,6 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
   endif()
 
   if(SWIFT_SWIFT_PARSER)
-    # Make sure we can find the early SwiftSyntax libraries.
-    target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
-
     # For the "end step" of bootstrapping configurations, we need to be
     # able to fall back to the SDK directory for libswiftCore et al.
     if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*")
@@ -832,8 +829,43 @@ macro(add_swift_lib_subdirectory name)
   add_llvm_subdirectory(SWIFT LIB ${name})
 endmacro()
 
+# Add a new Swift host executable.
+#
+# Usage:
+#   add_swift_host_tool(name
+#     [HAS_SWIFT_MODULES | DOES_NOT_USE_SWIFT]
+#     [THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY]
+#
+#     [BOOTSTRAPPING 0|1]
+#     [SWIFT_COMPONENT component]
+#     [LLVM_LINK_COMPONENTS comp1 ...]
+#     source1 [source2 source3 ...])
+#
+# name
+#   Name of the executable (e.g., swift-frontend).
+#
+# HAS_SWIFT_MODULES
+#   Whether to link with SwiftCompilerSources library
+#
+# DOES_NOT_USE_SWIFT
+#   Do not link with swift runtime
+#
+# THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY
+#   Opt-out of LLVM IR optimizations when linking ThinLTO with ld64
+#
+# BOOTSTRAPPING
+#   Bootstrapping stage.
+#
+# SWIFT_COMPONENT
+#   Installation component where this tool belongs to.
+#
+# LLVM_LINK_COMPONENTS
+#   LLVM components this library depends on.
+#
+# source1 ...
+#   Sources to add into this executable.
 function(add_swift_host_tool executable)
-  set(options HAS_SWIFT_MODULES THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY)
+  set(options HAS_SWIFT_MODULES DOES_NOT_USE_SWIFT THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY)
   set(single_parameter_options SWIFT_COMPONENT BOOTSTRAPPING)
   set(multiple_parameter_options LLVM_LINK_COMPONENTS)
 
@@ -889,12 +921,12 @@ function(add_swift_host_tool executable)
   endif()
 
   # Once the new Swift parser is linked in, every host tool has Swift modules.
-  if (SWIFT_SWIFT_PARSER)
+  if (SWIFT_SWIFT_PARSER AND NOT ASHT_DOES_NOT_USE_SWIFT)
     set(ASHT_HAS_SWIFT_MODULES ON)
   endif()
 
   if (ASHT_HAS_SWIFT_MODULES)
-      _add_swift_runtime_link_flags(${executable} "../lib" "${ASHT_BOOTSTRAPPING}")
+    _add_swift_runtime_link_flags(${executable} "../lib" "${ASHT_BOOTSTRAPPING}")
   endif()
 
   llvm_update_compile_flags(${executable})
@@ -967,7 +999,8 @@ function(add_swift_host_tool executable)
     swift_install_in_component(TARGETS ${executable}
                                RUNTIME
                                  DESTINATION bin
-                                 COMPONENT ${ASHT_SWIFT_COMPONENT})
+                                 COMPONENT ${ASHT_SWIFT_COMPONENT}
+    )
 
     swift_is_installing_component(${ASHT_SWIFT_COMPONENT} is_installing)
   endif()

diff --git a/cmake/modules/AddSwiftUnittests.cmake b/cmake/modules/AddSwiftUnittests.cmake
@@ -5,10 +5,32 @@ add_custom_target(SwiftUnitTests)
 
 set_target_properties(SwiftUnitTests PROPERTIES FOLDER "Tests")
 
+# Add a new Swift unit test executable.
+#
+# Usage:
+#   add_swift_unittest(name
+#     [IS_TARGET_TEST]
+#     source1 [source2 source3 ...])
+#
+# name
+#   Name of the test (e.g., SwiftASTTest).
+#
+# IS_TARGET_TEST
+#   Indicates this is a test for target libraries. Not host library.
+#   This avoids linking with toolchains stdlib.
+#
+# source1 ...
+#   Sources to add into this executable.
 function(add_swift_unittest test_dirname)
+  cmake_parse_arguments(ASU
+                        "IS_TARGET_TEST"
+                        ""
+                        ""
+                        ${ARGN})
+
   # *NOTE* Even though "add_unittest" does not have llvm in its name, it is a
   # function defined by AddLLVM.cmake.
-  add_unittest(SwiftUnitTests ${test_dirname} ${ARGN})
+  add_unittest(SwiftUnitTests ${test_dirname} ${ASU_UNPARSED_ARGUMENTS})
 
   set_target_properties(${test_dirname} PROPERTIES LINKER_LANGUAGE CXX)
 
@@ -89,7 +111,8 @@ function(add_swift_unittest test_dirname)
     endif()
   endif()
 
-  if (SWIFT_SWIFT_PARSER)
+  if (SWIFT_SWIFT_PARSER AND NOT ASU_IS_TARGET_TEST)
+    # Link to stdlib the compiler uses.
     _add_swift_runtime_link_flags(${test_dirname} "../../lib" "")
     set_property(TARGET ${test_dirname} PROPERTY BUILD_WITH_INSTALL_RPATH OFF)
   endif()

diff --git a/cmake/modules/SetRPATH.cmake b/cmake/modules/SetRPATH.cmake
@@ -0,0 +1,24 @@
+set(SWIFT_SET_RPATH_SCRIPT_FILE "${CMAKE_CURRENT_LIST_FILE}")
+
+function(swift_get_set_rpath_script_file out_var)
+  set(${out_var} "${SWIFT_SET_RPATH_SCRIPT_FILE}" PARENT_SCOPE)
+endfunction()
+
+# Actual RPATH_CHANGE operation to the file.
+function(_swift_set_rpath_impl file new_rpath)
+  # FIXME: Handle non-ELF files. We can't use RPATH_SET because it's only available CMake 3.21.0
+  execute_process(
+    COMMAND readelf -Wd "${file}"
+    COMMAND grep -Po "R(UN)?PATH.*\\[\\K[^\\]]*"
+    OUTPUT_VARIABLE current_rpath
+  )
+  string(STRIP "${current_rpath}" current_rpath)
+
+  # NOTE: RPATH_CHANGE is not documented, and works only for ELF and XCOFF.
+  file(RPATH_CHANGE FILE "${file}" OLD_RPATH "${current_rpath}" NEW_RPATH "${new_rpath}")
+endfunction()
+
+# For 'cmake -P <scirpt>'.
+if (SWIFT_SET_RPATH_FILE AND SWIFT_SET_RPATH_NEW_RPATH)
+  _swift_set_rpath_impl("${SWIFT_SET_RPATH_FILE}" "${SWIFT_SET_RPATH_NEW_RPATH}")
+endif()
diff --git a/cmake/modules/SwiftImplicitImport.cmake b/cmake/modules/SwiftImplicitImport.cmake
@@ -6,7 +6,7 @@ function(swift_supports_implicit_module module_name out_var)
       "${CMAKE_Swift_COMPILER}"
       -Xfrontend -disable-implicit-${module_name}-module-import
       -Xfrontend -parse-stdlib
-      -c - -o /dev/null
+      -parse -
     INPUT_FILE
       "${CMAKE_BINARY_DIR}/tmp/empty-check-${module_name}.swift"
     OUTPUT_QUIET ERROR_QUIET

diff --git a/cmake/modules/SwiftUtils.cmake b/cmake/modules/SwiftUtils.cmake
@@ -107,6 +107,16 @@ function(get_bootstrapping_swift_lib_dir bs_lib_dir bootstrapping)
     elseif("${bootstrapping}" STREQUAL "")
       get_bootstrapping_path(bs_lib_dir ${lib_dir} "1")
     endif()
+  elseif(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS")
+    if(SWIFT_HOST_VARIANT_SDK MATCHES "LINUX|ANDROID|OPENBSD|FREEBSD")
+      # Compiler's INSTALL_RPATH is set to libs in the build directory
+      # For building stdlib, use stdlib in the builder's resource directory
+      # because the runtime may not be built yet.
+      # FIXME: This assumes the ABI hasn't changed since the builder.
+      get_filename_component(swift_bin_dir ${CMAKE_Swift_COMPILER} DIRECTORY)
+      get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY)
+      set(bs_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
+    endif()
   endif()
   set(bs_lib_dir ${bs_lib_dir} PARENT_SCOPE)
 endfunction()

diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
@@ -33,7 +33,7 @@ if (SWIFT_SWIFT_PARSER)
   list(TRANSFORM SWIFT_SYNTAX_MODULES PREPEND "SwiftSyntax::"
        OUTPUT_VARIABLE SWIFT_SYNTAX_TARGETS)
 
-  set(SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR
+  set(SWIFT_SYNTAX_LIBRARIES_BUILD_DIR
       "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
   set(SWIFT_HOST_LIBRARIES_DEST_DIR
       "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/swift/host")
@@ -53,17 +53,35 @@ if (SWIFT_SWIFT_PARSER)
   # Copy over all of the shared libraries from earlyswiftsyntax so they can
   # be found via RPATH.
   foreach (sharedlib ${SWIFT_SYNTAX_SHARED_LIBRARIES})
+    set(add_origin_rpath)
+    if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+      # At runtime, use swiftCore in the current toolchain.
+      swift_get_set_rpath_script_file(setrpath_command)
+      set(add_origin_rpath COMMAND ${CMAKE_COMMAND}
+        "-DSWIFT_SET_RPATH_FILE=${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
+        "-DSWIFT_SET_RPATH_NEW_RPATH='$$ORIGIN:$$ORIGIN/../${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}'"
+        -P "${setrpath_command}"
+      )
+    endif()
+
     add_custom_command(
       OUTPUT "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
-      DEPENDS "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib}"
-      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}
+      DEPENDS "${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${sharedlib}"
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${sharedlib} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}
+      ${add_origin_rpath}
     )
 
     add_custom_target(copy_swiftSyntaxLibrary_${sharedlib}
       DEPENDS "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
       COMMENT "Copying ${sharedlib}"
     )
 
+    swift_install_in_component(
+      PROGRAMS "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
+      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host"
+      COMPONENT compiler
+    )
+
     add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxLibrary_${sharedlib})
   endforeach()
 
@@ -76,7 +94,7 @@ if (SWIFT_SWIFT_PARSER)
   foreach(module_dir ${SWIFT_SYNTAX_MODULE_DIRS})
     # Find all of the source module files.
     file(GLOB module_files
-         "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${module_dir}/*.swiftinterface")
+      "${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${module_dir}/*.swiftinterface")
 
     # Determine the destination module files.
     set(dest_module_files)
@@ -98,6 +116,12 @@ if (SWIFT_SWIFT_PARSER)
       COMMENT "Copying ${module_dir}"
     )
 
+    swift_install_in_component(
+      FILES ${dest_module_files}
+      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/${module_dir}"
+      COMPONENT compiler
+    )
+
     add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxModule_${module_dir})
   endforeach()
 

diff --git a/lib/Macros/CMakeLists.txt b/lib/Macros/CMakeLists.txt
@@ -31,6 +31,19 @@ function(add_swift_macro_library name)
   # Add the library.
   add_pure_swift_host_library(${name} SHARED ${ASML_SOURCES})
 
+  # Add rpath to 'lib/{platform}'
+  file(RELATIVE_PATH relpath_to_lib
+    "${SWIFT_HOST_PLUGINS_DEST_DIR}"
+    "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
+  )
+  _set_pure_swift_link_flags(${name} "${relpath_to_lib}")
+
+  # Add rpath to 'lib/host'
+  if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+    set_property(TARGET ${name}
+      APPEND PROPERTY INSTALL_RPATH "$ORIGIN/..")
+  endif()
+
   # If we don't have the Swift swift parser, bail out, because the above
   # add_pure_swift_host_library did nothing.
   if (NOT SWIFT_SWIFT_PARSER)
@@ -44,12 +57,14 @@ function(add_swift_macro_library name)
       LIBRARY_OUTPUT_DIRECTORY "${SWIFT_HOST_PLUGINS_DEST_DIR}"
   )
 
+  set(destination_dir "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins")
+
   swift_install_in_component(TARGETS ${name}
     LIBRARY
-      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins"
+      DESTINATION "${destination_dir}"
       COMPONENT compiler
     ARCHIVE
-      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins"
+      DESTINATION "${destination_dir}"
       COMPONENT compiler)
 
   # Export this macro plugin target.

@@ -801,7 +801,10 @@ function(_compile_swift_files
     set(swift_compiler_tool "${SWIFT_SOURCE_DIR}/utils/check-incremental" "${swift_compiler_tool}")
   endif()
 
-  if(SWIFTFILE_IS_STDLIB)
+  if(SWIFTFILE_IS_STDLIB OR
+     # Linux "hosttools" build require builder's runtime before building the runtime.
+     (BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS" AND SWIFT_HOST_VARIANT_SDK MATCHES "LINUX|ANDROID|OPENBSD|FREEBSD")
+  )
     get_bootstrapping_swift_lib_dir(bs_lib_dir "${SWIFTFILE_BOOTSTRAPPING}")
     if(bs_lib_dir)
       # When building the stdlib with bootstrapping, the compiler needs