Always build SwiftDriver and use it as default compiler driver

cmake/modules/SwiftUtils.cmake

@@ -152,6 +152,35 @@ function(swift_create_post_build_symlink target)
     COMMENT "${CS_COMMENT}")
 endfunction()
 
+# Once swift-frontend is built, if the standalone (early) swift-driver has been built,
+# we create a `swift-driver` symlink adjacent to the `swift` and `swiftc` executables
+# to ensure that `swiftc` forwards to the standalone driver when invoked.
+function(swift_create_early_driver_symlinks target)
+  # Early swift-driver is built adjacent to the compiler (swift build dir)
+  set(driver_bin_dir "${CMAKE_BINARY_DIR}/../earlyswiftdriver-${SWIFT_HOST_VARIANT}-${SWIFT_HOST_VARIANT_ARCH}/release/bin")
+  set(swift_bin_dir "${SWIFT_RUNTIME_OUTPUT_INTDIR}")
+  # If early swift-driver wasn't built, nothing to do here.
+  if(NOT EXISTS "${driver_bin_dir}/swift-driver" OR NOT EXISTS "${driver_bin_dir}/swift-help")
+      message(STATUS "Skipping creating early SwiftDriver symlinks - no early SwiftDriver build found.")
+      return()
+  endif()
+
+  message(STATUS "Creating early SwiftDriver symlinks.")
+  message(STATUS "From: ${driver_bin_dir}/swift-driver")
+  message(STATUS "To: ${swift_bin_dir}/swift-driver")
+  swift_create_post_build_symlink(swift-frontend
+    SOURCE "${driver_bin_dir}/swift-driver"
+    DESTINATION "${swift_bin_dir}/swift-driver"
+    COMMENT "Creating early SwiftDriver symlinks: swift-driver")
+
+  message(STATUS "From: ${driver_bin_dir}/swift-help")
+  message(STATUS "To: ${swift_bin_dir}/swift-help")
+  swift_create_post_build_symlink(swift-frontend
+    SOURCE "${driver_bin_dir}/swift-help"
+    DESTINATION "${swift_bin_dir}/swift-help"
+    COMMENT "Creating early SwiftDriver symlinks: swift-help")
+endfunction()
+
 function(dump_swift_vars)
   set(SWIFT_STDLIB_GLOBAL_CMAKE_CACHE)
   get_cmake_property(variableNames VARIABLES)

docs/DebuggingTheCompiler.md

@@ -36,6 +36,14 @@ benefit of all Swift developers.
         - [Bisecting on SIL optimizer pass counts to identify optimizer bugs](#bisecting-on-sil-optimizer-pass-counts-to-identify-optimizer-bugs)
         - [Using git-bisect in the presence of branch forwarding/feature branches](#using-git-bisect-in-the-presence-of-branch-forwardingfeature-branches)
         - [Reducing SIL test cases using bug_reducer](#reducing-sil-test-cases-using-bug_reducer)
+- [Debugging the Compiler Build](#debugging-the-compiler-build)
+    - [Build Dry Run](#build-dry-run)
+- [Debugging the Compiler Driver](#debugging-the-compiler-driver-build)
+    - [Swift Compiler Driver F.A.Q](#swift-compiler-driver-f.a.q.)
+    - [Building the compiler without using the standalone driver](#building-the-compiler-without-the-standalone-driver)
+    - [Invoking the compiler without forwarding to the standalone driver](#invoking-the-compiler-without-forwarding-to-the-standalone-driver)
+    - [Reproducing the Compiler Driver build steps](#reproducing-the-compiler-driver-build-steps)
+    - [Installing the Compiler Driver](#installing-the-compiler-driver)
 - [Debugging Swift Executables](#debugging-swift-executables)
     - [Determining the mangled name of a function in LLDB](#determining-the-mangled-name-of-a-function-in-lldb)
     - [Manually symbolication using LLDB](#manually-symbolication-using-lldb)
@@ -807,6 +815,131 @@ reducing SIL test cases by:
 For more information and a high level example, see:
 ./swift/utils/bug_reducer/README.md.
 
+# Debugging the Compiler Build
+
+## Build Dry Run
+
+A "dry-run" invocation of the `build-script` (using the `--dry-run` flag) will
+print the commands that would be executed in a given build, without executing
+them. A dry-run script invocation output can be used to inspect the build stages
+of a given `build-script` configuration, or create script corresponding to one
+such configuration.
+
+# Debugging the Compiler Driver
+
+The Swift compiler uses a standalone compiler-driver application written in
+Swift: [swift-driver](https://github.com/apple/swift-driver). When building the
+compiler using `build-script`, by default, the standalone driver will be built
+first, using the host toolchain, if the host toolchain contains a Swift
+compiler. If the host toolchain does not contain Swift, a warning is emitted and
+the legacy compiler-driver (integrated in the C++ code-base) will be used. In
+the future, a host toolchain containing a Swift compiler may become mandatory.
+Once the compiler is built, the compiler build directory (`swift-<OS>-<ARCH>`)
+is updated with a symlink to the standalone driver, ensuring calls to the build
+directory's `swift` and `swiftc` always forward to the standalone driver.
+
+For more information about the driver, see:
+[github.com/apple/swift-driver/blob/main/README.md](https://github.com/apple/swift-driver/blob/main/README.md)
+
+## Swift Compiler Driver F.A.Q.
+> What's the difference between invoking 'swiftc' vs. 'swift-driver' at the top
+  level?
+
+Today, `swift` and `swiftc` are symbolic links to the compiler binary
+(`swift-frontend`). Invoking `swiftc` causes the executable to detects that it
+is a compiler-driver invocation, and not a direct compiler-frontend invocation,
+by examining the invoked program's name. The compiler frontend can be invoked
+directly by invoking the `swift-frontend` executable, or passing in the
+`-frontend` option to `swiftc`.
+
+The standalone [Compiler Driver](https://github.com/apple/swift-driver) is
+installed as a separate `swift-driver` executable in the Swift toolchain's `bin`
+directory. When a user launches the compiler by invoking `swiftc`, the C++ based
+compiler executable forwards the invocation to the `swift-driver` executable if
+one is found alongside it. This forwarding mechanism is in-place temporarily, to
+allow for an easy fallback to the legacy driver via one of the two escape
+hatches:
+
+- `-disallow-use-new-driver` command line flag
+- `SWIFT_USE_OLD_DRIVER` environment variable
+
+If the user is to directly invoke the `swift-driver` executable, the behaviour
+should be the same as invoking the `swiftc` executable, but without the option
+for a legacy driver fallback.
+
+Once the legacy driver is deprecated, `swift` and `swiftc` executables will
+become symbolic links to the `swift-driver` executable directly.
+
+
+> Will 'swiftc ... -###' always print the same set of commands for the old/new
+  driver? Do they call 'swift-frontend' the same way?
+
+The standalone [Compiler Driver](https://github.com/apple/swift-driver) is meant
+to be a direct drop-in replacement for the C++-based legacy driver. It has the
+exact same command-line interface. The expectation is that its behaviour closely
+matches the legacy driver; however, during, and after the transition to the new
+driver being the default its behaviour may start to diverge from the legacy
+driver as par for the course of its evolution and gaining new features, etc.
+Today, broadly-speaking, sets of `swift-frontend` invocations generated by the
+two drivers are expected to be very similar.
+
+## Building the compiler without the standalone driver
+One can build the compiler that does not rely on the standalone driver and
+instead uses the legacy, built-in driver using the `build-script` option: 
+`--skip-early-swift-driver`.
+
+## Invoking the compiler without forwarding to the standalone driver
+The Swift compiler can currently be invoked in an execution mode that will use
+the legacy C++-based compiler driver using one of the following two options:
+- Passing `-disallow-use-new-driver` argument to the `swiftc` invocation
+- Setting the `SWIFT_USE_OLD_DRIVER` environment variable 
+
+## Reproducing the Compiler Driver build steps
+A "[dry-run](#build-dry-run)" invocation of the `build-script` can be used to
+examine the SwiftDriver build stage and commands, without executing it. For
+example:
+```
+$ utils/build-script --release-debuginfo --dry-run
++ mkdir -p /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert
+--- Building earlyswiftdriver ---
++ /SwiftWorkspace/swift-driver/Utilities/build-script-helper.py build --package-path /SwiftWorkspace/swift-driver --build-path /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/earlyswiftdriver-macosx-x86_64 --configuration release --toolchain /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr --ninja-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/ninja --cmake-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/cmake --local_compiler_build
+Building the standard library for: swift-test-stdlib-macosx-x86_64
+...
+```
+One of the first steps is an invocation of the driver's
+`build-script-helper.py` script which specifies that the driver us to be built
+(`build`) using the host toolchain (`--toolchain`) to a specified location
+(`--build-path`). 
+
+## Installing the Compiler Driver
+In order to create a Swift compiler installation (`--install-swift`), the
+standalone driver must be built as a separate build product using the
+*just-built* Swift compiler and toolchain (the ones built in the same
+`build-script` invocation, preceeding the SwiftDriver build product). The
+additional build product is added to the build by specifying the
+`--swift-driver` option of the `build-script`. The driver product is istalled
+into the resulting toolchain installation by specifying the
+`--install-swift-driver` option of the `build-script`.
+
+Note, a "dry-run" `build-script` invocation when installing the standalone
+driver will demonstrate the commands required to build and install the driver as
+a standalone build product:
+```
+$ utils/build-script --release-debuginfo --dry-run --swift-driver --install-swift-driver
+...
+--- Cleaning swiftdriver ---
++ /SwiftWorkspace/swift-driver/Utilities/build-script-helper.py clean --package-path /SwiftWorkspace/swift-driver --build-path /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/swiftdriver-macosx-x86_64 --configuration release --toolchain /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/toolchain-macosx-x86_64/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr --ninja-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/ninja --cmake-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/cmake
+--- Building swiftdriver ---
++ /SwiftWorkspace/swift-driver/Utilities/build-script-helper.py build --package-path /SwiftWorkspace/swift-driver --build-path /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/swiftdriver-macosx-x86_64 --configuration release --toolchain /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/toolchain-macosx-x86_64/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr --ninja-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/ninja --cmake-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/cmake
+--- Installing swiftdriver ---
++ /SwiftWorkspace/swift-driver/Utilities/build-script-helper.py install --package-path /SwiftWorkspace/swift-driver --build-path /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/swiftdriver-macosx-x
+86_64 --configuration release --toolchain /SwiftWorkspace/build/Ninja-RelWithDebInfoAssert/toolchain-macosx-x86_64/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr --ninja-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/ninja --cmake-bin /Applications/Xcode.app/Contents/Developer/usr/local/bin/cmake
+```
+These invocations of the driver's `build-script-helper.py` script specify the
+individual build actions (`clean`, `build`, `install`), the product build path
+(`--build-path`), and the *just-built* toolchain which should be used
+(`--toolchain`).
+
 # Debugging Swift Executables
 
 One can use the previous tips for debugging the Swift compiler with Swift

test/CMakeLists.txt

@@ -188,6 +188,7 @@ set(TEST_SUBSETS
     only_validation
     only_long
     only_stress
+    only_early_swiftdriver
 )
 
 if(NOT "${COVERAGE_DB}" STREQUAL "")
@@ -353,6 +354,7 @@ foreach(SDK ${SWIFT_SDKS})
              (test_subset STREQUAL "validation") OR
              (test_subset STREQUAL "only_long") OR
              (test_subset STREQUAL "only_stress") OR
+             (test_subset STREQUAL "only_early_swiftdriver") OR
              (test_subset STREQUAL "all"))
             list(APPEND directories "${test_bin_dir}")
           endif()

test/Unit/lit.cfg

@@ -63,7 +63,7 @@ elif swift_test_subset == 'only_stress':
     #   Currently those tests are very fast so it doesn't matter much.
     pass
 else:
-    lit_config.fatal("Unknown test mode %r" % swift_test_subset)
+    lit_config.fatal("Unknown test subset %r" % swift_test_subset)
 
 # test_source_root: The root path where tests are located.
 # test_exec_root: The root path where tests should be run.

test/lit.cfg

@@ -631,6 +631,10 @@ elif swift_test_subset == 'only_stress':
     config.available_features.add("stress_test")
     config.limit_to_features.add("stress_test")
     config.limit_to_features.discard("executable_test")
+elif swift_test_subset == 'only_early_swiftdriver':
+    # Point this subset at a driver-specific set of tests. These are the known reduced subset
+    # of tests to verify the basic functionality of the standalone (early) swift-driver.
+    config.test_source_root = os.path.join(config.test_source_root, 'Driver', 'Dependencies')
 else:
     lit_config.fatal("Unknown test mode %r" % swift_test_subset)
 
@@ -640,6 +644,13 @@ if 'swift_evolve' in lit_config.params:
 if not 'swift_driver' in lit_config.params:
     config.available_features.add("cplusplus_driver")
 
+# Check if we need to run lit tests using the legacy driver or the new driver
+# The default for existing test runs is to use the legacy driver.
+# The new driver is tested separately.
+if swift_test_subset != 'only_early_swiftdriver' and\
+   os.environ.get('SWIFT_FORCE_TEST_NEW_DRIVER') is None:
+    config.environment['SWIFT_USE_OLD_DRIVER'] = '1'
+
 # Enable benchmark testing when the binary is found (has fully qualified path).
 if config.benchmark_o != 'Benchmark_O':
     config.available_features.add('benchmark')

tools/driver/CMakeLists.txt

@@ -16,6 +16,10 @@ target_link_libraries(swift-frontend
                         swiftSymbolGraphGen
                         LLVMBitstreamReader)
 
+# Create a `swift-driver` symlinks adjacent to the `swift-frontend` executable
+# to ensure that `swiftc` forwards to the standalone driver when invoked.
+swift_create_early_driver_symlinks(swift-frontend)
+
 swift_create_post_build_symlink(swift-frontend
   SOURCE "swift-frontend${CMAKE_EXECUTABLE_SUFFIX}"
   DESTINATION "swift${CMAKE_EXECUTABLE_SUFFIX}"