[CMake] Replace early swift-syntax with FetchContent

Use FetchContent to include swift-syntax directly in swift. This can be
thought of as an `add_subdirectory` for a directory outside the root.

The default build directory will be `_deps/swiftsyntax-subbuild/`, though
the modules and shared libraries will be built in `lib/swift/host` by
passing down `SWIFT_HOST_LIBRARIES_DEST_DIR` to avoid copying them as we
were doing previously.

Author: bnbarham

CMakeLists.txt

@@ -94,6 +94,7 @@ include(CMakeDependentOption)
 include(CheckLanguage)
 include(GNUInstallDirs)
 include(SwiftImplicitImport)
+include(FetchContent)
 
 # Enable Swift for the host compiler build if we have the language. It is
 # optional until we have a bootstrap story.
@@ -686,10 +687,9 @@ if(CMAKE_C_COMPILER_ID MATCHES Clang)
   add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-Werror=c++98-compat-extra-semi>)
 endif()
 
-# Make sure we know where swift-syntax is because we need it to build the parser.
-if(NOT EXISTS "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}")
-  message(SEND_ERROR "swift-syntax is required to build the Swift compiler. Please run update-checkout or specify SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE")
-endif()
+option(SWIFT_BUILD_SWIFT_SYNTAX
+  "Enable building swift syntax"
+  FALSE)
 
 # Use dispatch as the system scheduler by default.
 # For convenience, we set this to false when concurrency is disabled.
@@ -831,7 +831,7 @@ elseif(BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*")
   else()
     set(BOOTSTRAPPING_MODE "HOSTTOOLS")
   endif()
-elseif(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS" OR SWIFT_SWIFT_PARSER)
+elseif(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS" OR SWIFT_BUILD_SWIFT_SYNTAX)
   # We are building using a pre-installed host toolchain but not bootstrapping
   # the Swift modules. This happens when building using 'build-tooling-libs'
   # where we haven't built a new Swift compiler. Use the Swift compiler from the
@@ -945,19 +945,6 @@ if(XCODE)
   set(SWIFT_SDKS "OSX")
 endif()
 
-# When we have the early SwiftSyntax build, we can include its parser.
-if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR)
-  set(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS
-    ${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake)
-  if(NOT EXISTS "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS}")
-    message(STATUS "Skipping Swift Swift parser integration due to missing early SwiftSyntax")
-  else()
-    set(SWIFT_SWIFT_PARSER TRUE)
-    include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS})
-  endif()
-endif()
-
-
 # FIXME: the parameters we specify in SWIFT_SDKS are lacking architecture specifics,
 # so we need to hard-code it. For example, the SDK for Android is just 'ANDROID',
 # and we have to specify SWIFT_SDK_ANDROID_ARCHITECTURES separately.
@@ -1158,13 +1145,29 @@ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND NOT CMAKE_CROSSCOMPILING)
   set(CMAKE_OSX_DEPLOYMENT_TARGET "")
 endif()
 
+set(SWIFT_HOST_MODULE_TRIPLE "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_ARCH_${SWIFT_HOST_VARIANT_ARCH}_MODULE}")
+set(SWIFT_HOST_LIBRARIES_DEST_DIR "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/swift/host")
+if(SWIFT_BUILD_SWIFT_SYNTAX)
+  if(NOT EXISTS "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}")
+    message(SEND_ERROR "swift-syntax is required to build the Swift compiler. Please run update-checkout or specify SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE")
+  endif()
+
+  set(BUILD_SHARED_LIBS_OLD "${BUILD_SHARED_LIBS}")
+  set(BUILD_SHARED_LIBS ON)
+  FetchContent_Declare(SwiftSyntax
+    SOURCE_DIR "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}"
+  )
+  FetchContent_MakeAvailable(SwiftSyntax)
+  set(BUILD_SHARED_LIBS "${BUILD_SHARED_LIBS_OLD}")
+endif()
+
 if(SWIFT_INCLUDE_TOOLS)
   message(STATUS "Building host Swift tools for ${SWIFT_HOST_VARIANT_SDK} ${SWIFT_HOST_VARIANT_ARCH}")
   message(STATUS "  Build type:     ${CMAKE_BUILD_TYPE}")
   message(STATUS "  Assertions:     ${LLVM_ENABLE_ASSERTIONS}")
   message(STATUS "  LTO:            ${SWIFT_TOOLS_ENABLE_LTO}")
   message(STATUS "  Bootstrapping:  ${BOOTSTRAPPING_MODE}")
-  message(STATUS "  Swift parser:   ${SWIFT_SWIFT_PARSER}")
+  message(STATUS "  Swift parser:   ${SWIFT_BUILD_SWIFT_SYNTAX}")
   message(STATUS "")
 else()
   message(STATUS "Not building host Swift tools")

cmake/modules/AddPureSwift.cmake

@@ -103,7 +103,7 @@ endfunction()
 # source1 ...
 #   Sources to add into this library.
 function(add_pure_swift_host_library name)
-  if (NOT SWIFT_SWIFT_PARSER)
+  if (NOT SWIFT_BUILD_SWIFT_SYNTAX)
     message(STATUS "Not building ${name} because swift-syntax is not available")
     return()
   endif()
@@ -172,13 +172,15 @@ function(add_pure_swift_host_library name)
 
   # Make sure we can use the host libraries.
   target_include_directories(${name} PUBLIC
-    ${SWIFT_HOST_LIBRARIES_DEST_DIR})
+    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
+  target_link_directories(${name} PUBLIC
+    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
 
   if(APSHL_EMIT_MODULE)
     # Determine where Swift modules will be built and installed.
 
-    set(module_triple ${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_ARCH_${SWIFT_HOST_VARIANT_ARCH}_MODULE})
-    set(module_dir ${SWIFT_HOST_LIBRARIES_DEST_DIR})
+    set(module_triple "${SWIFT_HOST_MODULE_TRIPLE}")
+    set(module_dir "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
     set(module_base "${module_dir}/${name}.swiftmodule")
     set(module_file "${module_base}/${module_triple}.swiftmodule")
     set(module_interface_file "${module_base}/${module_triple}.swiftinterface")
@@ -210,14 +212,20 @@ function(add_pure_swift_host_library name)
         >)
   endif()
 
+  if(LLVM_USE_LINKER)
+    target_link_options(${name} PRIVATE
+      "-use-ld=${LLVM_USE_LINKER}"
+    )
+  endif()
+
   # Export this target.
   set_property(GLOBAL APPEND PROPERTY SWIFT_EXPORTS ${name})
 endfunction()
 
 # Add a new "pure" Swift host tool.
 #
 # "Pure" Swift host tools can only contain Swift code, and will be built
-# with the host compiler. 
+# with the host compiler.
 #
 # Usage:
 #   add_pure_swift_host_tool(name
@@ -238,7 +246,7 @@ endfunction()
 # source1 ...
 #   Sources to add into this tool.
 function(add_pure_swift_host_tool name)
-  if (NOT SWIFT_SWIFT_PARSER)
+  if (NOT SWIFT_BUILD_SWIFT_SYNTAX)
     message(STATUS "Not building ${name} because swift-syntax is not available")
     return()
   endif()
@@ -290,7 +298,15 @@ function(add_pure_swift_host_tool name)
 
   # Make sure we can use the host libraries.
   target_include_directories(${name} PUBLIC
-    ${SWIFT_HOST_LIBRARIES_DEST_DIR})
+    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
+  target_link_directories(${name} PUBLIC
+    "${SWIFT_HOST_LIBRARIES_DEST_DIR}")
+
+  if(LLVM_USE_LINKER)
+    target_link_options(${name} PRIVATE
+      "-use-ld=${LLVM_USE_LINKER}"
+    )
+  endif()
 
   # Export this target.
   set_property(GLOBAL APPEND PROPERTY SWIFT_EXPORTS ${name})

cmake/modules/AddSwift.cmake

@@ -439,7 +439,7 @@ endfunction()
 
 function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
   if(NOT BOOTSTRAPPING_MODE)
-    if (SWIFT_SWIFT_PARSER)
+    if (SWIFT_BUILD_SWIFT_SYNTAX)
       set(ASRLF_BOOTSTRAPPING_MODE "HOSTTOOLS")
     else()
       return()
@@ -572,10 +572,7 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
     endif()
   endif()
 
-  if(SWIFT_SWIFT_PARSER)
-    # Make sure we can find the early SwiftSyntax libraries.
-    target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
-
+  if(SWIFT_BUILD_SWIFT_SYNTAX)
     # For the "end step" of bootstrapping configurations on Darwin, need to be
     # able to fall back to the SDK directory for libswiftCore et al.
     if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*")
@@ -646,7 +643,7 @@ function(add_swift_host_library name)
   translate_flags(ASHL "${options}")
 
   # Once the new Swift parser is linked, everything has Swift modules.
-  if (SWIFT_SWIFT_PARSER AND ASHL_SHARED)
+  if (SWIFT_BUILD_SWIFT_SYNTAX AND ASHL_SHARED)
     set(ASHL_HAS_SWIFT_MODULES ON)
   endif()
 
@@ -692,7 +689,7 @@ function(add_swift_host_library name)
 
   add_library(${name} ${libkind} ${ASHL_SOURCES})
 
-  target_link_directories(${name} PUBLIC ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+  target_link_directories(${name} PUBLIC "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}")
 
   # Respect LLVM_COMMON_DEPENDS if it is set.
   #
@@ -879,7 +876,7 @@ function(add_swift_host_tool executable)
   endif()
 
   # Once the new Swift parser is linked in, every host tool has Swift modules.
-  if (SWIFT_SWIFT_PARSER)
+  if (SWIFT_BUILD_SWIFT_SYNTAX)
     set(ASHT_HAS_SWIFT_MODULES ON)
   endif()
 
@@ -918,7 +915,7 @@ function(add_swift_host_tool executable)
     endif()
   endif()
 
-  if(SWIFT_SWIFT_PARSER)
+  if(SWIFT_BUILD_SWIFT_SYNTAX)
     set(extra_relative_rpath "")
     if(NOT ${ASHT_BOOTSTRAPPING} STREQUAL "")
       if (${executable} MATCHES "-bootstrapping")

cmake/modules/AddSwiftUnittests.cmake

@@ -81,7 +81,7 @@ function(add_swift_unittest test_dirname)
     endif()
   endif()
 
-  if (SWIFT_SWIFT_PARSER)
+  if (SWIFT_BUILD_SWIFT_SYNTAX)
     _add_swift_runtime_link_flags(${test_dirname} "../../lib" "")
     set_property(TARGET ${test_dirname} PROPERTY BUILD_WITH_INSTALL_RPATH OFF)
   endif()

lib/AST/CMakeLists.txt

@@ -153,10 +153,10 @@ target_link_libraries(swiftAST INTERFACE
   clangAPINotes
   clangBasic)
 
-if(SWIFT_SWIFT_PARSER)
+if(SWIFT_BUILD_SWIFT_SYNTAX)
   target_compile_definitions(swiftAST
     PRIVATE
-    SWIFT_SWIFT_PARSER
+    SWIFT_BUILD_SWIFT_SYNTAX
   )
 endif()
 

lib/ASTGen/CMakeLists.txt

@@ -25,13 +25,13 @@ add_pure_swift_host_library(swiftASTGen STATIC
   DEPENDENCIES
     swiftAST
   SWIFT_DEPENDENCIES
-    SwiftSyntax::SwiftBasicFormat
-    SwiftSyntax::SwiftDiagnostics
-    SwiftSyntax::SwiftOperators
-    SwiftSyntax::SwiftParser
-    SwiftSyntax::SwiftParserDiagnostics
-    SwiftSyntax::SwiftSyntax
-    SwiftSyntax::SwiftSyntaxMacros
-    SwiftSyntax::SwiftSyntaxMacroExpansion
+    SwiftBasicFormat
+    SwiftDiagnostics
+    SwiftOperators
+    SwiftParser
+    SwiftParserDiagnostics
+    SwiftSyntax
+    SwiftSyntaxMacros
+    SwiftSyntaxMacroExpansion
     swiftLLVMJSON
 )

lib/CMakeLists.txt

@@ -15,99 +15,6 @@ list(APPEND LLVM_COMMON_DEPENDS intrinsics_gen clang-tablegen-targets)
 # Add generated Swift Syntax headers to global dependencies.
 list(APPEND LLVM_COMMON_DEPENDS swift-ast-generated-headers)
 
-# Set up for linking against swift-syntax.
-if (SWIFT_SWIFT_PARSER)
-  # Set up linking against the swift-syntax modules.
-  # Link against the swift-syntax modules.
-  set(SWIFT_SYNTAX_MODULES
-    SwiftBasicFormat
-    SwiftParser
-    SwiftParserDiagnostics
-    SwiftDiagnostics
-    SwiftSyntax
-    SwiftOperators
-    SwiftSyntaxBuilder
-    SwiftSyntaxMacros
-    SwiftSyntaxMacroExpansion
-    SwiftCompilerPluginMessageHandling
-  )
-
-  # Compute the list of SwiftSyntax targets that we will link against.
-  list(TRANSFORM SWIFT_SYNTAX_MODULES PREPEND "SwiftSyntax::"
-       OUTPUT_VARIABLE SWIFT_SYNTAX_TARGETS)
-
-  set(SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR
-      "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
-  set(SWIFT_HOST_LIBRARIES_DEST_DIR
-      "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/swift/host")
-
-  # Determine the SwiftSyntax shared library files that were built as
-  # part of earlyswiftsyntax.
-  list(TRANSFORM SWIFT_SYNTAX_MODULES PREPEND ${CMAKE_SHARED_LIBRARY_PREFIX}
-       OUTPUT_VARIABLE SWIFT_SYNTAX_SHARED_LIBRARIES)
-  list(TRANSFORM SWIFT_SYNTAX_SHARED_LIBRARIES APPEND
-       ${CMAKE_SHARED_LIBRARY_SUFFIX}
-       OUTPUT_VARIABLE SWIFT_SYNTAX_SHARED_LIBRARIES)
-
-  # Interface library to collect swiftinterfaces and swiftmodules from
-  # SwiftSyntax
-  add_library(swiftSyntaxLibraries INTERFACE)
-
-  # Copy over all of the shared libraries from earlyswiftsyntax so they can
-  # be found via RPATH.
-  foreach (sharedlib ${SWIFT_SYNTAX_SHARED_LIBRARIES})
-    add_custom_command(
-      OUTPUT "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
-      DEPENDS "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib}"
-      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}
-    )
-
-    add_custom_target(copy_swiftSyntaxLibrary_${sharedlib}
-      DEPENDS "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
-      COMMENT "Copying ${sharedlib}"
-    )
-
-    add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxLibrary_${sharedlib})
-  endforeach()
-
-  # Copy all of the Swift modules from earlyswiftsyntax so they can be found
-  # in the same relative place within the build directory as in the final
-  # toolchain.
-  list(TRANSFORM SWIFT_SYNTAX_MODULES APPEND ".swiftmodule"
-       OUTPUT_VARIABLE SWIFT_SYNTAX_MODULE_DIRS)
-
-  foreach(module_dir ${SWIFT_SYNTAX_MODULE_DIRS})
-    # Find all of the source module files.
-    file(GLOB module_files
-         "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${module_dir}/*.swiftinterface")
-
-    # Determine the destination module files.
-    set(dest_module_files)
-    foreach(full_module_file ${module_files})
-        get_filename_component(module_file ${full_module_file} NAME)
-        list(APPEND dest_module_files
-            "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${module_dir}/${module_file}")
-    endforeach()
-
-    add_custom_command(
-      OUTPUT ${dest_module_files}
-      DEPENDS ${module_files}
-      COMMAND ${CMAKE_COMMAND} -E make_directory ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${module_dir}
-      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${module_files} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${module_dir}/
-    )
-
-    add_custom_target(copy_swiftSyntaxModule_${module_dir}
-      DEPENDS ${dest_module_files}
-      COMMENT "Copying ${module_dir}"
-    )
-
-    add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxModule_${module_dir})
-  endforeach()
-
-  # Add copied SwiftSyntax libraries to global dependencies.
-  list(APPEND LLVM_COMMON_DEPENDS swiftSyntaxLibraries)
-endif()
-
 add_subdirectory(APIDigester)
 add_subdirectory(AST)
 add_subdirectory(ASTGen)

lib/Frontend/CMakeLists.txt

@@ -37,9 +37,9 @@ target_link_libraries(swiftFrontend PRIVATE
 
 set_swift_llvm_is_available(swiftFrontend)
 
-if (SWIFT_SWIFT_PARSER)
+if (SWIFT_BUILD_SWIFT_SYNTAX)
   target_compile_definitions(swiftFrontend
     PRIVATE
-    SWIFT_SWIFT_PARSER
+    SWIFT_BUILD_SWIFT_SYNTAX
     )
 endif()