diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..2a9df2dac --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +xcode_trim_whitespace_on_empty_lines = true diff --git a/.gitignore b/.gitignore index c54f4ce9c..5d1d81717 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ xcuserdata/ .swiftpm .*.sw? +.vscode/launch.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ab9fb9b3..3492f63a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,61 +8,91 @@ cmake_minimum_required(VERSION 3.19.6) +if(POLICY CMP0077) + cmake_policy(SET CMP0077 NEW) +endif() + +if(POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) +endif() + +project(SwiftDriver + LANGUAGES C CXX Swift) + +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +set(CMAKE_MACOSX_RPATH YES) +set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules) -project(SwiftDriver LANGUAGES C Swift) +set(CMAKE_Swift_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY MultiThreadedDLL) +set(CMAKE_Swift_LANGUAGE_VERSION 5) +set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift) -set(SWIFT_VERSION 5) -set(CMAKE_Swift_LANGUAGE_VERSION ${SWIFT_VERSION}) -if(CMAKE_VERSION VERSION_LESS 3.16) - add_compile_options($<$:-swift-version$${SWIFT_VERSION}>) - set(CMAKE_LINK_LIBRARY_FLAG "-l") +# Generate build-ids +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" + AND NOT CMAKE_SYSTEM_NAME STREQUAL "Windows") + add_link_options("LINKER:--build-id=sha1") endif() # ensure Swift compiler can find _CSwiftScan add_compile_options($<$:-I$${CMAKE_CURRENT_SOURCE_DIR}/Sources/CSwiftScan/include>) -set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift) +option(BUILD_SHARED_LIBS "Build shared libraries by default" YES) +option(SWIFT_DRIVER_BUILD_TOOLS "Build makeOption" NO) -if(CMAKE_VERSION VERSION_LESS 3.16 AND CMAKE_SYSTEM_NAME STREQUAL Windows) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -else() - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) -endif() -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# Toolchain Vended Dependencies +find_package(dispatch QUIET) +find_package(Foundation QUIET) -option(BUILD_SHARED_LIBS "Build shared libraries by default" YES) +include(FetchContent) -set(CMAKE_MACOSX_RPATH YES) -if(CMAKE_VERSION VERSION_LESS 3.17) - if(NOT CMAKE_SYSTEM_NAME STREQUAL Windows) - set(CMAKE_EXECUTABLE_RUNTIME_Swift_FLAG "-Xlinker -rpath -Xlinker ") - set(CMAKE_SHARED_LIBRARY_RUNTIME_Swift_FLAG "-Xlinker -rpath -Xlinker ") - - if(CMAKE_SYSTEM_NAME STREQUAL Darwin) - set(CMAKE_EXECUTABLE_RUNTIME_Swift_FLAG_SEP "") - set(CMAKE_SHARED_LIBRARY_RUNTIME_Swift_FLAG_SEP "") - else() - set(CMAKE_EXECUTABLE_RUNTIME_Swift_FLAG_SEP ":") - set(CMAKE_SHARED_LIBRARY_RUNTIME_Swift_FLAG_SEP ":") - endif() - endif() -endif() +set(VendoredDependencies) -find_package(TSC CONFIG REQUIRED) +find_package(ArgumentParser CONFIG) +if(NOT ArgumentParser_FOUND) + message("-- Vending swift-argument-parser") + FetchContent_Declare(ArgumentParser + GIT_REPOSITORY https://github.com/apple/swift-argument-parser + GIT_TAG 1.2.3) + list(APPEND VendoredDependencies ArgumentParser) +endif() find_package(LLBuild CONFIG) if(NOT LLBuild_FOUND) - find_package(LLBuild REQUIRED) + if(APPLE) + find_package(LLBuild REQUIRED) + else() + message("-- Vending swift-llbuild") + set(LLBUILD_SUPPORT_BINDINGS Swift) + FetchContent_Declare(LLBuild + GIT_REPOSITORY https://github.com/apple/swift-llbuild + GIT_TAG main) + list(APPEND VendoredDependencies LLBuild) + endif() endif() -find_package(dispatch QUIET) -find_package(Foundation QUIET) -find_package(Yams CONFIG REQUIRED) -find_package(ArgumentParser CONFIG REQUIRED) -find_package(SwiftSystem CONFIG REQUIRED) +find_package(TSC CONFIG) +if(NOT TSC_FOUND) + message("-- Vending swift-tools-support-core") + FetchContent_Declare(ToolsSupportCore + GIT_REPOSITORY https://github.com/apple/swift-tools-support-core + GIT_TAG main) + list(APPEND VendoredDependencies ToolsSupportCore) +endif() + +set(_SD_SAVED_BUILD_TESTING ${BUILD_TESTING}) +set(_SD_SAVED_BUILD_EXAMPLES ${BUILD_EXAMPLES}) + +set(BUILD_TESTING NO) +set(BUILD_EXAMPLES NO) + +FetchContent_MakeAvailable(${VendoredDependencies}) + +set(BUILD_TESTING ${_SD_SAVED_BUILD_TESTING}) +set(BUILD_EXAMPLES ${_SD_SAVED_BUILD_EXAMPLES}) add_subdirectory(Sources) add_subdirectory(cmake/modules) diff --git a/Package.resolved b/Package.resolved index eac64def5..9606eb451 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,52 +1,41 @@ { - "object": { - "pins": [ - { - "package": "swift-argument-parser", - "repositoryURL": "https://github.com/apple/swift-argument-parser.git", - "state": { - "branch": null, - "revision": "fee6933f37fde9a5e12a1e4aeaa93fe60116ff2a", - "version": "1.2.2" - } - }, - { - "package": "llbuild", - "repositoryURL": "https://github.com/apple/swift-llbuild.git", - "state": { - "branch": "main", - "revision": "efb176d866e3e99855a5b799da224a9f95058f2e", - "version": null - } - }, - { - "package": "swift-system", - "repositoryURL": "https://github.com/apple/swift-system.git", - "state": { - "branch": null, - "revision": "836bc4557b74fe6d2660218d56e3ce96aff76574", - "version": "1.1.1" - } - }, - { - "package": "swift-tools-support-core", - "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", - "state": { - "branch": "main", - "revision": "7c8bcf3eab7286855a2eb0cd0df103ea2761e259", - "version": null - } - }, - { - "package": "Yams", - "repositoryURL": "https://github.com/jpsim/Yams.git", - "state": { - "branch": null, - "revision": "01835dc202670b5bb90d07f3eae41867e9ed29f6", - "version": "5.0.1" - } + "pins" : [ + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "41982a3656a71c768319979febd796c6fd111d5c", + "version" : "1.5.0" } - ] - }, - "version": 1 + }, + { + "identity" : "swift-llbuild", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-llbuild.git", + "state" : { + "branch" : "main", + "revision" : "02db743e7fd4ba241b78207309ddb3c9c2ec5f3f" + } + }, + { + "identity" : "swift-toolchain-sqlite", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-toolchain-sqlite", + "state" : { + "revision" : "bb8321a7eea3830af401a1501c7c8cc27ded6793", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-tools-support-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-tools-support-core.git", + "state" : { + "branch" : "main", + "revision" : "4074f4db0971328c441fc1621c673937b9ca3b08" + } + } + ], + "version" : 2 } diff --git a/Package.swift b/Package.swift index 16d72b8bb..8b7da6ee0 100644 --- a/Package.swift +++ b/Package.swift @@ -10,6 +10,8 @@ if let deploymentTarget = ProcessInfo.processInfo.environment["SWIFTPM_MACOS_DEP macOSPlatform = .macOS(.v12) } +let swiftToolsSupportCoreLibName = (ProcessInfo.processInfo.environment["SWIFT_DRIVER_USE_STSC_DYLIB"] == nil) ? "SwiftToolsSupport-auto": "SwiftToolsSupport" + let package = Package( name: "swift-driver", platforms: [ @@ -51,9 +53,8 @@ let package = Package( name: "SwiftDriver", dependencies: [ "SwiftOptions", - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + .product(name: swiftToolsSupportCoreLibName, package: "swift-tools-support-core"), "CSwiftScan", - .product(name: "Yams", package: "yams"), ], exclude: ["CMakeLists.txt"]), @@ -62,14 +63,14 @@ let package = Package( name: "SwiftDriverExecution", dependencies: [ "SwiftDriver", - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core") + .product(name: swiftToolsSupportCoreLibName, package: "swift-tools-support-core") ], exclude: ["CMakeLists.txt"]), /// Driver tests. .testTarget( name: "SwiftDriverTests", - dependencies: ["SwiftDriver", "SwiftDriverExecution", "TestUtilities"]), + dependencies: ["SwiftDriver", "SwiftDriverExecution", "TestUtilities", "ToolingTestShim"]), /// IncrementalImport tests .testTarget( @@ -77,7 +78,7 @@ let package = Package( dependencies: [ "IncrementalTestFramework", "TestUtilities", - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + .product(name: swiftToolsSupportCoreLibName, package: "swift-tools-support-core"), ]), .target( @@ -93,11 +94,16 @@ let package = Package( dependencies: ["SwiftDriver", "SwiftDriverExecution"], path: "Tests/TestUtilities"), + .target( + name: "ToolingTestShim", + dependencies: ["SwiftDriver"], + path: "Tests/ToolingTestShim"), + /// The options library. .target( name: "SwiftOptions", dependencies: [ - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + .product(name: swiftToolsSupportCoreLibName, package: "swift-tools-support-core"), ], exclude: ["CMakeLists.txt"]), .testTarget( @@ -116,11 +122,11 @@ let package = Package( dependencies: [ "SwiftOptions", .product(name: "ArgumentParser", package: "swift-argument-parser"), - .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), + .product(name: swiftToolsSupportCoreLibName, package: "swift-tools-support-core"), ], exclude: ["CMakeLists.txt"]), - /// The help executable. + /// Build SDK Interfaces tool executable. .executableTarget( name: "swift-build-sdk-interfaces", dependencies: ["SwiftDriver", "SwiftDriverExecution"], @@ -129,19 +135,22 @@ let package = Package( /// The `makeOptions` utility (for importing option definitions). .executableTarget( name: "makeOptions", - dependencies: []), + dependencies: [], + // Do not enforce checks for LLVM's ABI-breaking build settings. + // makeOptions runtime uses some header-only code from LLVM's ADT classes, + // but we do not want to link libSupport into the executable. + cxxSettings: [.unsafeFlags(["-DLLVM_DISABLE_ABI_BREAKING_CHECKS_ENFORCING=1"])]), ], - cxxLanguageStandard: .cxx14 + cxxLanguageStandard: .cxx17 ) if ProcessInfo.processInfo.environment["SWIFT_DRIVER_LLBUILD_FWK"] == nil { if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { package.dependencies += [ - .package(url: "https://github.com/apple/swift-llbuild.git", branch: "main"), + .package(url: "https://github.com/swiftlang/swift-llbuild.git", branch: "main"), ] package.targets.first(where: { $0.name == "SwiftDriverExecution" })!.dependencies += [ .product(name: "llbuildSwift", package: "swift-llbuild"), - .product(name: "llbuild", package: "swift-llbuild"), ] } else { // In Swift CI, use a local path to llbuild to interoperate with tools @@ -151,24 +160,21 @@ if ProcessInfo.processInfo.environment["SWIFT_DRIVER_LLBUILD_FWK"] == nil { ] package.targets.first(where: { $0.name == "SwiftDriverExecution" })!.dependencies += [ .product(name: "llbuildSwift", package: "llbuild"), - .product(name: "llbuild", package: "llbuild"), ] } } if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { package.dependencies += [ - .package(url: "https://github.com/apple/swift-tools-support-core.git", branch: "main"), - .package(url: "https://github.com/jpsim/Yams.git", .upToNextMinor(from: "5.0.0")), + .package(url: "https://github.com/swiftlang/swift-tools-support-core.git", branch: "main"), // The 'swift-argument-parser' version declared here must match that // used by 'swift-package-manager' and 'sourcekit-lsp'. Please coordinate // dependency version changes here with those projects. - .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.2.2")), - ] + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.4.0"), + ] } else { package.dependencies += [ .package(path: "../swift-tools-support-core"), - .package(path: "../yams"), .package(path: "../swift-argument-parser"), ] } diff --git a/README.md b/README.md index fa9833545..bdf36e052 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Swift Compiler Driver -Swift's compiler driver is a program that coordinates the compilation of Swift source code into various compiled results: executables, libraries, object files, Swift modules and interfaces, etc. It is the program one invokes from the command line to build Swift code (i.e., `swift` or `swiftc`) and is often invoked on the developer's behalf by a build system such as the [Swift Package Manager (SwiftPM)](https://github.com/apple/swift-package-manager) or Xcode's build system. +Swift's compiler driver is a program that coordinates the compilation of Swift source code into various compiled results: executables, libraries, object files, Swift modules and interfaces, etc. It is the program one invokes from the command line to build Swift code (i.e., `swift` or `swiftc`) and is often invoked on the developer's behalf by a build system such as the [Swift Package Manager (SwiftPM)](https://github.com/swiftlang/swift-package-manager) or Xcode's build system. The `swift-driver` project is a new implementation of the Swift compiler driver that is intended to replace the [existing driver](https://github.com/apple/swift/tree/main/lib/Driver) with a more extensible, maintainable, and robust code base. The specific goals of this project include: * A maintainable, robust, and flexible Swift code base * Library-based architecture that allows better integration with build tools -* Leverage existing Swift build technologies ([SwiftPM](https://github.com/apple/swift-package-manager), [llbuild](https://github.com/apple/swift-llbuild)) +* Leverage existing Swift build technologies ([SwiftPM](https://github.com/swiftlang/swift-package-manager), [llbuild](https://github.com/apple/swift-llbuild)) * A platform for experimenting with more efficient build models for Swift, including compile servers and unifying build graphs across different driver invocations ## Getting Started @@ -21,22 +21,12 @@ On most platforms you can build using: $ swift build ``` -However, on Windows, some additional work must be done by the developer. - -Due to the default version of swift-tools-support-core that `Package.resolved` references, we must first update the package dependencies. - -``` -swift package update -``` - -Then, we can build the package using: +However, on Windows, because Swift Package Manager does not differentiate between C/C++ and Swift targets and uses the Swift driver as the linker driver we must link in the Swift runtime into all targets manually: ```cmd -swift build -Xcc -I -Xcc "%SystemDrive%\Library\sqlite-3.38.0\usr\include" -Xlinker -L -Xlinker "%SystemDrive%\Library\sqlite-3.38.0\usr\lib" -Xlinker "%SDKROOT%\usr\lib\swift\windows\x86_64\swiftCore.lib" +swift build -Xlinker "%SDKROOT%\usr\lib\swift\windows\x86_64\swiftCore.lib" ``` -Because SQLite3 is a system library dependency, and there is no singular header and library search path, the developer must specify that. The path to SQLite3 may need to be adjusted if the library is not located at the specified location. Additionally, because Swift Package Manager does not differentiate between C/C++ and Swift targets and uses the Swift driver as the linker driver we must link in the Swift runtime into all targets manually. - To use `swift-driver` in place of the existing Swift driver, create a symbolic link from `swift` and `swiftc` to `swift-driver`: ``` @@ -64,13 +54,11 @@ all with CMake: ``` cmake -B -G Ninja -DLLBUILD_SUPPORT_BINDINGS="Swift" -DCMAKE_OSX_ARCHITECTURES=x86_64 ``` -* [swift-system](https://github.com/apple/swift-system) * [swift-argument-parser](https://github.com/apple/swift-argument-parser) -* [Yams](https://github.com/jpsim/Yams) Once those dependencies have built, build `swift-driver` itself: ``` -cmake -B -G Ninja -DTSC_DIR=/cmake/modules -DLLBuild_DIR=/cmake/modules -DYams_DIR=/cmake/modules -DArgumentParser_DIR= +cmake -B -G Ninja -DTSC_DIR=/cmake/modules -DLLBuild_DIR=/cmake/modules -DArgumentParser_DIR= cmake --build ``` @@ -156,7 +144,7 @@ $ apt-get install libncurses-dev be found, e.g.: ``` -$ swift build -Xcc -I/path/to/build/Ninja-ReleaseAssert/swift-.../include --product makeOptions +$ swift build -Xcc -I/path/to/build/Ninja-Release/swift-.../include -Xcc -I/path/to/build/Ninja-Release/llvm-.../include -Xcc -I/path/to/source/llvm-project/llvm/include --product makeOptions ``` Then, run `makeOptions` and redirect the output to overwrite `Options.swift`: @@ -217,3 +205,26 @@ $SDKROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/De * *SWIFT_EXEC*: teach `swift-build-sdk-interfaces` about where to find the Swift compiler to use * *-O*: the output directory for all binary modules built from textual interfaces * *-log-path*: where to dump log files when fatal error happens + + +## Contributing to swift-driver + +Contributions to swift-driver are welcomed and encouraged! Please see the +[Contributing to Swift guide](https://swift.org/contributing/). + +Before submitting the pull request, please make sure you have [tested your + changes](https://github.com/apple/swift/blob/main/docs/ContinuousIntegration.md) + and that they follow the Swift project [guidelines for contributing + code](https://swift.org/contributing/#contributing-code). + +To be a truly great community, [Swift.org](https://swift.org/) needs to welcome +developers from all walks of life, with different backgrounds, and with a wide +range of experience. A diverse and friendly community will have more great +ideas, more unique perspectives, and produce more great code. We will work +diligently to make the Swift community welcoming to everyone. + +To give clarity of what is expected of our members, Swift has adopted the +code of conduct defined by the Contributor Covenant. This document is used +across many open source communities, and we think it articulates our values +well. For more, see the [Code of Conduct](https://swift.org/code-of-conduct/). + diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 2be6a1bdd..601d93e99 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -13,3 +13,7 @@ add_subdirectory(SwiftDriverExecution) add_subdirectory(swift-build-sdk-interfaces) add_subdirectory(swift-driver) add_subdirectory(swift-help) + +if(SWIFT_DRIVER_BUILD_TOOLS) + add_subdirectory(makeOptions) +endif() diff --git a/Sources/CSwiftScan/CMakeLists.txt b/Sources/CSwiftScan/CMakeLists.txt index 20f31d3c8..fea0189d4 100644 --- a/Sources/CSwiftScan/CMakeLists.txt +++ b/Sources/CSwiftScan/CMakeLists.txt @@ -8,3 +8,4 @@ add_library(CSwiftScan STATIC CSwiftScanImpl.c) +set_property(GLOBAL APPEND PROPERTY SWIFTDRIVER_EXPORTS CSwiftScan) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 79ea98bd4..f6e8e00cd 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -17,8 +17,8 @@ #include #include -#define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 5 +#define SWIFTSCAN_VERSION_MAJOR 2 +#define SWIFTSCAN_VERSION_MINOR 1 //=== Public Scanner Data Types -------------------------------------------===// @@ -41,9 +41,11 @@ typedef enum { typedef struct swiftscan_module_details_s *swiftscan_module_details_t; typedef struct swiftscan_dependency_info_s *swiftscan_dependency_info_t; +typedef struct swiftscan_link_library_info_s *swiftscan_link_library_info_t; typedef struct swiftscan_dependency_graph_s *swiftscan_dependency_graph_t; typedef struct swiftscan_import_set_s *swiftscan_import_set_t; typedef struct swiftscan_diagnostic_info_s *swiftscan_diagnostic_info_t; +typedef struct swiftscan_source_location_s *swiftscan_source_location_t; typedef enum { SWIFTSCAN_DIAGNOSTIC_SEVERITY_ERROR = 0, @@ -59,18 +61,10 @@ typedef struct { swiftscan_dependency_info_t *modules; size_t count; } swiftscan_dependency_set_t; - -//=== Batch Scan Input Specification --------------------------------------===// - -typedef struct swiftscan_batch_scan_entry_s *swiftscan_batch_scan_entry_t; -typedef struct { - swiftscan_batch_scan_entry_t *modules; - size_t count; -} swiftscan_batch_scan_input_t; typedef struct { - swiftscan_dependency_graph_t *results; + swiftscan_link_library_info_t *link_libraries; size_t count; -} swiftscan_batch_scan_result_t; +} swiftscan_link_library_set_t; //=== Scanner Invocation Specification ------------------------------------===// @@ -78,17 +72,15 @@ typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t; typedef void *swiftscan_scanner_t; //=== CAS/Caching Specification -------------------------------------------===// -typedef struct swiftscan_cas_s *swiftscan_cas_t; typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t; - -typedef enum { - SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0, - SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1, - SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2, - SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3, - SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4, - SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5 -} swiftscan_output_kind_t; +typedef struct swiftscan_cas_s *swiftscan_cas_t; +typedef struct swiftscan_cached_compilation_s *swiftscan_cached_compilation_t; +typedef struct swiftscan_cached_output_s *swiftscan_cached_output_t; +typedef struct swiftscan_cache_replay_instance_s + *swiftscan_cache_replay_instance_t; +typedef struct swiftscan_cache_replay_result_s *swiftscan_cache_replay_result_t; +typedef struct swiftscan_cache_cancellation_token_s + *swiftscan_cache_cancellation_token_t; //=== libSwiftScan Functions ------------------------------------------------===// @@ -99,6 +91,8 @@ typedef struct { (*swiftscan_dependency_graph_get_main_module_name)(swiftscan_dependency_graph_t); swiftscan_dependency_set_t * (*swiftscan_dependency_graph_get_dependencies)(swiftscan_dependency_graph_t); + swiftscan_diagnostic_set_t * + (*swiftscan_dependency_graph_get_diagnostics)(swiftscan_dependency_graph_t); //=== Dependency Module Info Functions ------------------------------------===// swiftscan_string_ref_t @@ -109,9 +103,19 @@ typedef struct { (*swiftscan_module_info_get_source_files)(swiftscan_dependency_info_t); swiftscan_string_set_t * (*swiftscan_module_info_get_direct_dependencies)(swiftscan_dependency_info_t); + swiftscan_link_library_set_t * + (*swiftscan_module_info_get_link_libraries)(swiftscan_dependency_graph_t); swiftscan_module_details_t (*swiftscan_module_info_get_details)(swiftscan_dependency_info_t); + //=== Link Library Info Functions ------------------------------------===// + swiftscan_string_ref_t + (*swiftscan_link_library_info_get_link_name)(swiftscan_link_library_info_t); + bool + (*swiftscan_link_library_info_get_is_framework)(swiftscan_link_library_info_t); + bool + (*swiftscan_link_library_info_get_should_force_load)(swiftscan_link_library_info_t); + //=== Dependency Module Info Details Functions ----------------------------===// swiftscan_dependency_info_kind_t (*swiftscan_module_detail_get_kind)(swiftscan_module_details_t); @@ -131,16 +135,22 @@ typedef struct { (*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t); - swiftscan_string_set_t * - (*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t); bool (*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t); + swiftscan_string_set_t * + (*swiftscan_swift_textual_detail_get_swift_source_import_module_dependencies)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_textual_detail_get_user_module_version)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_textual_detail_get_chained_bridging_header_path)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_textual_detail_get_chained_bridging_header_content)(swiftscan_module_details_t); //=== Swift Binary Module Details query APIs ------------------------------===// swiftscan_string_ref_t @@ -149,12 +159,18 @@ typedef struct { (*swiftscan_swift_binary_detail_get_module_doc_path)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_binary_detail_get_module_source_info_path)(swiftscan_module_details_t); - swiftscan_string_set_t * - (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); + swiftscan_string_ref_t + (*swiftscan_swift_binary_detail_get_header_dependency)(swiftscan_module_details_t); bool (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t); + swiftscan_string_set_t * + (*swiftscan_swift_binary_detail_get_header_dependency_module_dependencies)(swiftscan_module_details_t); + + //=== Swift Binary Module Details deprecated APIs--------------------------===// + swiftscan_string_set_t * + (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); //=== Swift Placeholder Module Details query APIs -------------------------===// swiftscan_string_ref_t @@ -171,36 +187,14 @@ typedef struct { (*swiftscan_clang_detail_get_context_hash)(swiftscan_module_details_t); swiftscan_string_set_t * (*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t); - swiftscan_string_set_t * - (*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t); - //=== Batch Scan Input Functions ------------------------------------------===// - swiftscan_batch_scan_input_t * - (*swiftscan_batch_scan_input_create)(void); - void - (*swiftscan_batch_scan_input_set_modules)(swiftscan_batch_scan_input_t *, int, swiftscan_batch_scan_entry_t *); - - //=== Batch Scan Entry Functions ------------------------------------------===// - swiftscan_batch_scan_entry_t - (*swiftscan_batch_scan_entry_create)(void); - void - (*swiftscan_batch_scan_entry_set_module_name)(swiftscan_batch_scan_entry_t, const char *); - void - (*swiftscan_batch_scan_entry_set_arguments)(swiftscan_batch_scan_entry_t, const char *); - void - (*swiftscan_batch_scan_entry_set_is_swift)(swiftscan_batch_scan_entry_t, bool); - swiftscan_string_ref_t - (*swiftscan_batch_scan_entry_get_module_name)(swiftscan_batch_scan_entry_t); - swiftscan_string_ref_t - (*swiftscan_batch_scan_entry_get_arguments)(swiftscan_batch_scan_entry_t); - bool - (*swiftscan_batch_scan_entry_get_is_swift)(swiftscan_batch_scan_entry_t); - //=== Prescan Result Functions --------------------------------------------===// swiftscan_string_set_t * (*swiftscan_import_set_get_imports)(swiftscan_import_set_t); + swiftscan_diagnostic_set_t * + (*swiftscan_import_set_get_diagnostics)(swiftscan_import_set_t); //=== Scanner Invocation Functions ----------------------------------------===// swiftscan_scan_invocation_t @@ -225,14 +219,6 @@ typedef struct { (*swiftscan_dependency_graph_dispose)(swiftscan_dependency_graph_t); void (*swiftscan_import_set_dispose)(swiftscan_import_set_t); - void - (*swiftscan_batch_scan_entry_dispose)(swiftscan_batch_scan_entry_t); - void - (*swiftscan_batch_scan_input_dispose)(swiftscan_batch_scan_input_t *); - void - (*swiftscan_batch_scan_result_dispose)(swiftscan_batch_scan_result_t *); - void - (*swiftscan_scan_invocation_dispose)(swiftscan_scan_invocation_t); //=== Target Info Functions-------- ---------------------------------------===// swiftscan_string_ref_t @@ -250,10 +236,6 @@ typedef struct { void (*swiftscan_scanner_dispose)(swiftscan_scanner_t); swiftscan_dependency_graph_t (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t); - swiftscan_batch_scan_result_t * - (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t, - swiftscan_batch_scan_input_t *, - swiftscan_scan_invocation_t); swiftscan_import_set_t (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t); @@ -266,33 +248,111 @@ typedef struct { (*swiftscan_diagnostic_get_message)(swiftscan_diagnostic_info_t); swiftscan_diagnostic_severity_t (*swiftscan_diagnostic_get_severity)(swiftscan_diagnostic_info_t); + swiftscan_source_location_t + (*swiftscan_diagnostic_get_source_location)(swiftscan_diagnostic_info_t); void (*swiftscan_diagnostics_set_dispose)(swiftscan_diagnostic_set_t*); + void + (*swiftscan_scan_invocation_dispose)(swiftscan_scan_invocation_t); - //=== Scanner Cache Functions ---------------------------------------------===// - void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path); - bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path); - void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner); + //=== Source Location -----------------------------------------------------===// + swiftscan_string_ref_t + (*swiftscan_source_location_get_buffer_identifier)(swiftscan_source_location_t); + int64_t + (*swiftscan_source_location_get_line_number)(swiftscan_source_location_t); + int64_t + (*swiftscan_source_location_get_column_number)(swiftscan_source_location_t); //=== Scanner CAS Operations ----------------------------------------------===// swiftscan_cas_options_t (*swiftscan_cas_options_create)(void); + int64_t (*swiftscan_cas_get_ondisk_size)(swiftscan_cas_t, + swiftscan_string_ref_t *error); + bool (*swiftscan_cas_set_ondisk_size_limit)(swiftscan_cas_t, + int64_t size_limit, + swiftscan_string_ref_t *error); + bool (*swiftscan_cas_prune_ondisk_data)(swiftscan_cas_t, + swiftscan_string_ref_t *error); void (*swiftscan_cas_options_dispose)(swiftscan_cas_options_t options); void (*swiftscan_cas_options_set_ondisk_path)(swiftscan_cas_options_t options, const char *path); void (*swiftscan_cas_options_set_plugin_path)(swiftscan_cas_options_t options, const char *path); - bool (*swiftscan_cas_options_set_option)(swiftscan_cas_options_t options, - const char *name, const char *value, - swiftscan_string_ref_t *error); + bool (*swiftscan_cas_options_set_plugin_option)( + swiftscan_cas_options_t options, const char *name, const char *value, + swiftscan_string_ref_t *error); swiftscan_cas_t (*swiftscan_cas_create_from_options)( swiftscan_cas_options_t options, swiftscan_string_ref_t *error); void (*swiftscan_cas_dispose)(swiftscan_cas_t cas); swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas, uint8_t *data, unsigned size, swiftscan_string_ref_t *error); - swiftscan_string_ref_t (*swiftscan_compute_cache_key)( - swiftscan_cas_t cas, int argc, const char *argv, const char *input, - swiftscan_output_kind_t, swiftscan_string_ref_t *error); + swiftscan_string_ref_t (*swiftscan_cache_compute_key)( + swiftscan_cas_t cas, int argc, const char **argv, const char *input, + swiftscan_string_ref_t *error); + swiftscan_string_ref_t (*swiftscan_cache_compute_key_from_input_index)( + swiftscan_cas_t cas, int argc, const char **argv, unsigned input_index, + swiftscan_string_ref_t *error); + + //=== Scanner Caching Query/Replay Operations -----------------------------===// + swiftscan_cached_compilation_t (*swiftscan_cache_query)( + swiftscan_cas_t cas, const char *key, bool globally, + swiftscan_string_ref_t *error); + void (*swiftscan_cache_query_async)( + swiftscan_cas_t cas, const char *key, bool globally, void *ctx, + void (*callback)(void *ctx, swiftscan_cached_compilation_t, + swiftscan_string_ref_t error), + swiftscan_cache_cancellation_token_t *); + + + unsigned (*swiftscan_cached_compilation_get_num_outputs)( + swiftscan_cached_compilation_t); + swiftscan_cached_output_t (*swiftscan_cached_compilation_get_output)( + swiftscan_cached_compilation_t, unsigned idx); + bool (*swiftscan_cached_compilation_is_uncacheable)( + swiftscan_cached_compilation_t); + void (*swiftscan_cached_compilation_make_global_async)( + swiftscan_cached_compilation_t, void *ctx, + void (*callback)(void *ctx, swiftscan_string_ref_t error), + swiftscan_cache_cancellation_token_t *); + void (*swiftscan_cached_compilation_dispose)(swiftscan_cached_compilation_t); + + bool (*swiftscan_cached_output_load)(swiftscan_cached_output_t, + swiftscan_string_ref_t *error); + void (*swiftscan_cached_output_load_async)( + swiftscan_cached_output_t, void *ctx, + void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error), + swiftscan_cache_cancellation_token_t *); + bool (*swiftscan_cached_output_is_materialized)(swiftscan_cached_output_t); + swiftscan_string_ref_t (*swiftscan_cached_output_get_casid)( + swiftscan_cached_output_t); + swiftscan_string_ref_t (*swiftscan_cached_output_get_name)( + swiftscan_cached_output_t); + void (*swiftscan_cached_output_dispose)(swiftscan_cached_output_t); + + void (*swiftscan_cache_action_cancel)(swiftscan_cache_cancellation_token_t); + void (*swiftscan_cache_cancellation_token_dispose)( + swiftscan_cache_cancellation_token_t); + + void (*swiftscan_cache_download_cas_object_async)( + swiftscan_cas_t, const char *id, void *ctx, + void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error), + swiftscan_cache_cancellation_token_t *); + + swiftscan_cache_replay_instance_t (*swiftscan_cache_replay_instance_create)( + int argc, const char **argv, swiftscan_string_ref_t *error); + void (*swiftscan_cache_replay_instance_dispose)( + swiftscan_cache_replay_instance_t); + + swiftscan_cache_replay_result_t (*swiftscan_cache_replay_compilation)( + swiftscan_cache_replay_instance_t, swiftscan_cached_compilation_t, + swiftscan_string_ref_t *error); + + swiftscan_string_ref_t (*swiftscan_cache_replay_result_get_stdout)( + swiftscan_cache_replay_result_t); + swiftscan_string_ref_t (*swiftscan_cache_replay_result_get_stderr)( + swiftscan_cache_replay_result_t); + void (*swiftscan_cache_replay_result_dispose)( + swiftscan_cache_replay_result_t); } swiftscan_functions_t; diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index d7785e768..d6594845b 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -7,7 +7,6 @@ # See http://swift.org/CONTRIBUTORS.txt for Swift project authors add_library(SwiftDriver - "ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift" "ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift" "ExplicitModuleBuilds/ModuleDependencyScanning.swift" "ExplicitModuleBuilds/SerializableModuleArtifacts.swift" @@ -18,6 +17,7 @@ add_library(SwiftDriver SwiftScan/DependencyGraphBuilder.swift SwiftScan/Loader.swift SwiftScan/SwiftScan.swift + SwiftScan/SwiftScanCAS.swift Driver/CompilerMode.swift Driver/DebugInfo.swift @@ -72,7 +72,6 @@ add_library(SwiftDriver Jobs/APIDigesterJobs.swift Jobs/AutolinkExtractJob.swift - Jobs/BackendJob.swift Jobs/CommandLineArguments.swift Jobs/CompileJob.swift Jobs/DarwinToolchain+LinkerSupport.swift @@ -89,6 +88,7 @@ add_library(SwiftDriver Jobs/MergeModuleJob.swift Jobs/ModuleWrapJob.swift Jobs/Planning.swift + Jobs/PrintSupportedFeaturesJob.swift Jobs/PrintTargetInfoJob.swift Jobs/ReplJob.swift Jobs/SwiftHelpIntroJob.swift @@ -131,8 +131,6 @@ target_link_libraries(SwiftDriver PUBLIC TSCUtility SwiftOptions) target_link_libraries(SwiftDriver PRIVATE - CYaml - Yams CSwiftScan) set_property(GLOBAL APPEND PROPERTY SWIFTDRIVER_EXPORTS SwiftDriver) diff --git a/Sources/SwiftDriver/Driver/DebugInfo.swift b/Sources/SwiftDriver/Driver/DebugInfo.swift index 92c131501..2f8bdd859 100644 --- a/Sources/SwiftDriver/Driver/DebugInfo.swift +++ b/Sources/SwiftDriver/Driver/DebugInfo.swift @@ -41,9 +41,12 @@ } } - // The format of debug information. + /// The format of debug information. public let format: Format + /// The DWARF standard version to be produced. + public let dwarfVersion: UInt8 + /// The level of debug information. public let level: Level? diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 31e430b57..9e856531c 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -12,14 +12,15 @@ import SwiftOptions import class Dispatch.DispatchQueue - -import TSCBasic // <<< import class TSCBasic.DiagnosticsEngine +import class TSCBasic.UnknownLocation import enum TSCBasic.ProcessEnv +import func TSCBasic.withTemporaryDirectory import protocol TSCBasic.DiagnosticData import protocol TSCBasic.FileSystem import protocol TSCBasic.OutputByteStream import struct TSCBasic.AbsolutePath +import struct TSCBasic.ByteString import struct TSCBasic.Diagnostic import struct TSCBasic.FileInfo import struct TSCBasic.RelativePath @@ -69,13 +70,10 @@ public struct Driver { case optionRequiresAnother(String, String) // Explicit Module Build Failures case malformedModuleDependency(String, String) - case missingPCMArguments(String) case missingModuleDependency(String) case missingContextHashOnSwiftDependency(String) case dependencyScanningFailure(Int, String) case missingExternalDependency(String) - // Compiler Caching Failures - case unsupportedConfigurationForCaching(String) public var description: String { switch self { @@ -125,8 +123,6 @@ public struct Driver { // Explicit Module Build Failures case .malformedModuleDependency(let moduleName, let errorDescription): return "Malformed Module Dependency: \(moduleName), \(errorDescription)" - case .missingPCMArguments(let moduleName): - return "Missing extraPcmArgs to build Clang module: \(moduleName)" case .missingModuleDependency(let moduleName): return "Missing Module Dependency Info: \(moduleName)" case .missingContextHashOnSwiftDependency(let moduleName): @@ -137,8 +133,6 @@ public struct Driver { return "unable to load output file map '\(path)': \(error)" case .missingExternalDependency(let moduleName): return "Missing External dependency info for module: \(moduleName)" - case .unsupportedConfigurationForCaching(let reason): - return "unsupported configuration for -cache-compile-job: \(reason)" case .baselineGenerationRequiresTopLevelModule(let arg): return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'" case .optionRequiresAnother(let first, let second): @@ -161,6 +155,10 @@ public struct Driver { /// Whether we are using the driver as the integrated driver via libSwiftDriver public let integratedDriver: Bool + /// If true, the driver instance is executed in the context of a + /// Swift compiler image which contains symbols normally queried from a libSwiftScan instance. + internal let compilerIntegratedTooling: Bool + /// The file system which we should interact with. @_spi(Testing) public let fileSystem: FileSystem @@ -223,8 +221,10 @@ public struct Driver { /// Should use file lists for inputs (number of inputs exceeds `fileListThreshold`). let shouldUseInputFileList: Bool - /// VirtualPath for shared all sources file list. `nil` if unused. - @_spi(Testing) public let allSourcesFileList: VirtualPath? + /// VirtualPath for shared all sources file list. `nil` if unused. This is used as a cache for + /// the file list computed during CompileJob creation and only holds valid to be query by tests + /// after planning to build. + @_spi(Testing) public var allSourcesFileList: VirtualPath? = nil /// The mode in which the compiler will execute. @_spi(Testing) public let compilerMode: CompilerMode @@ -256,6 +256,9 @@ public struct Driver { /// The information about the module to produce. @_spi(Testing) public let moduleOutputInfo: ModuleOutputInfo + /// Information about the target variant module to produce if applicable + @_spi(Testing) public let variantModuleOutputInfo: ModuleOutputInfo? + /// Name of the package containing a target module or file. @_spi(Testing) public let packageName: String? @@ -271,10 +274,50 @@ public struct Driver { let enableCaching: Bool let useClangIncludeTree: Bool + /// CAS instance used for compilation. + @_spi(Testing) public var cas: SwiftScanCAS? = nil + + /// Is swift caching enabled. + lazy var isCachingEnabled: Bool = { + return enableCaching && isFeatureSupported(.compilation_caching) + }() + + /// Scanner prefix mapping. + let scannerPrefixMap: [AbsolutePath: AbsolutePath] + let scannerPrefixMapSDK: AbsolutePath? + let scannerPrefixMapToolchain: AbsolutePath? + lazy var prefixMapping: [(AbsolutePath, AbsolutePath)] = { + var mapping: [(AbsolutePath, AbsolutePath)] = scannerPrefixMap.map { + return ($0.key, $0.value) + } + do { + guard isFrontendArgSupported(.scannerPrefixMap) else { + return [] + } + if let sdkMapping = scannerPrefixMapSDK, + let sdkPath = absoluteSDKPath { + mapping.append((sdkPath, sdkMapping)) + } + if let toolchainMapping = scannerPrefixMapToolchain { + let toolchainPath = try toolchain.executableDir.parentDirectory // usr + .parentDirectory // toolchain + mapping.append((toolchainPath, toolchainMapping)) + } + // The mapping needs to be sorted so the mapping is determinisitic. + // The sorting order is reversed so /tmp/tmp is preferred over /tmp in remapping. + return mapping.sorted { $0.0 > $1.0 } + } catch { + return mapping.sorted { $0.0 > $1.0 } + } + }() + /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil + /// The graph of explicit module dependencies of this module, if the driver has planned an explicit module build. + public private(set) var intermoduleDependencyGraph: InterModuleDependencyGraph? = nil + /// The path of the SDK. public var absoluteSDKPath: AbsolutePath? { guard let path = frontendTargetInfo.sdkPath?.path else { @@ -292,11 +335,43 @@ public struct Driver { } } + /// If PCH job is needed. + let producePCHJob: Bool + + /// Original ObjC Header passed from command-line + let originalObjCHeaderFile: VirtualPath.Handle? + + + /// Enable bridging header chaining. + let bridgingHeaderChaining: Bool + /// The path to the imported Objective-C header. - let importedObjCHeader: VirtualPath.Handle? + lazy var importedObjCHeader: VirtualPath.Handle? = { + assert(explicitDependencyBuildPlanner != nil || + !parsedOptions.hasArgument(.driverExplicitModuleBuild) || + !inputFiles.contains { $0.type == .swift }, + "should not be queried before scanning") + let chainedBridgingHeader = try? explicitDependencyBuildPlanner?.getChainedBridgingHeaderFile() + return try? computeImportedObjCHeader(&parsedOptions, compilerMode: compilerMode, + chainedBridgingHeader: chainedBridgingHeader) ?? originalObjCHeaderFile + }() + + /// The directory to emit PCH file. + lazy var bridgingPrecompiledHeaderOutputDir: VirtualPath? = { + return try? computePrecompiledBridgingHeaderDir(&parsedOptions, + compilerMode: compilerMode) + }() /// The path to the pch for the imported Objective-C header. - let bridgingPrecompiledHeader: VirtualPath.Handle? + lazy var bridgingPrecompiledHeader: VirtualPath.Handle? = { + let contextHash = try? explicitDependencyBuildPlanner?.getMainModuleContextHash() + return computeBridgingPrecompiledHeader(&parsedOptions, + compilerMode: compilerMode, + importedObjCHeader: importedObjCHeader, + outputFileMap: outputFileMap, + outputDirectory: bridgingPrecompiledHeaderOutputDir, + contextHash: contextHash) + }() /// Path to the dependencies file. let dependenciesFilePath: VirtualPath.Handle? @@ -325,14 +400,213 @@ public struct Driver { /// Path to the TBD file (text-based dylib). let tbdPath: VirtualPath.Handle? - /// Path to the module documentation file. - let moduleDocOutputPath: VirtualPath.Handle? + /// Target-specific supplemental output file paths + struct SupplementalModuleTargetOutputPaths { + /// Path to the module documentation file. + let moduleDocOutputPath: VirtualPath.Handle? + + /// Path to the Swift interface file. + let swiftInterfacePath: VirtualPath.Handle? + + /// Path to the Swift private interface file. + let swiftPrivateInterfacePath: VirtualPath.Handle? + + /// Path to the Swift package interface file. + let swiftPackageInterfacePath: VirtualPath.Handle? + + /// Path to the Swift module source information file. + let moduleSourceInfoPath: VirtualPath.Handle? + + /// Path to the emitted API descriptor file. + let apiDescriptorFilePath: VirtualPath.Handle? + + /// Path to the emitted ABI descriptor file. + let abiDescriptorFilePath: TypedVirtualPath? + } + + private static func computeModuleOutputPaths( + _ parsedOptions: inout ParsedOptions, + moduleName: String, + packageName: String?, + moduleOutputInfo: ModuleOutputInfo, + compilerOutputType: FileType?, + compilerMode: CompilerMode, + emitModuleSeparately: Bool, + outputFileMap: OutputFileMap?, + projectDirectory: VirtualPath.Handle?, + apiDescriptorDirectory: VirtualPath?, + supportedFrontendFeatures: Set, + target: FrontendTargetInfo.Target, + isVariant: Bool) throws -> SupplementalModuleTargetOutputPaths { + struct SupplementalPathOptions { + let moduleDocPath: Option + let sourceInfoPath: Option + let apiDescriptorPath: Option + let abiDescriptorPath: Option + let moduleInterfacePath: Option + let privateInterfacePath: Option + let packageInterfacePath: Option + + static let targetPathOptions = SupplementalPathOptions( + moduleDocPath: .emitModuleDocPath, + sourceInfoPath: .emitModuleSourceInfoPath, + apiDescriptorPath: .emitApiDescriptorPath, + abiDescriptorPath: .emitAbiDescriptorPath, + moduleInterfacePath: .emitModuleInterfacePath, + privateInterfacePath: .emitPrivateModuleInterfacePath, + packageInterfacePath: .emitPackageModuleInterfacePath) + + static let variantTargetPathOptions = SupplementalPathOptions( + moduleDocPath: .emitVariantModuleDocPath, + sourceInfoPath: .emitVariantModuleSourceInfoPath, + apiDescriptorPath: .emitVariantApiDescriptorPath, + abiDescriptorPath: .emitVariantAbiDescriptorPath, + moduleInterfacePath: .emitVariantModuleInterfacePath, + privateInterfacePath: .emitVariantPrivateModuleInterfacePath, + packageInterfacePath: .emitVariantPackageModuleInterfacePath) + } + + let pathOptions: SupplementalPathOptions = isVariant ? .variantTargetPathOptions : .targetPathOptions + + let moduleDocOutputPath = try Self.computeModuleDocOutputPath( + &parsedOptions, + moduleOutputPath: moduleOutputInfo.output?.outputPath, + outputOption: pathOptions.moduleDocPath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name) + + let moduleSourceInfoPath = try Self.computeModuleSourceInfoOutputPath( + &parsedOptions, + moduleOutputPath: moduleOutputInfo.output?.outputPath, + outputOption: pathOptions.sourceInfoPath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name, + projectDirectory: projectDirectory) + + // --------------------- + // ABI Descriptor Path + func computeABIDescriptorFilePath(target: FrontendTargetInfo.Target, + features: Set) -> TypedVirtualPath? { + guard features.contains(KnownCompilerFeature.emit_abi_descriptor.rawValue) else { + return nil + } + // Emit the descriptor only on platforms where Library Evolution is + // supported + guard target.triple.isDarwin || parsedOptions.hasArgument(.enableLibraryEvolution) else { + return nil + } + guard let moduleOutput = moduleOutputInfo.output else { + return nil + } + + guard let path = try? VirtualPath.lookup(moduleOutput.outputPath) + .replacingExtension(with: .jsonABIBaseline) else { + return nil + } + return TypedVirtualPath(file: path.intern(), type: .jsonABIBaseline) + } + let abiDescriptorFilePath = computeABIDescriptorFilePath(target: target, + features: supportedFrontendFeatures) + + // --------------------- + // API Descriptor Path + let apiDescriptorFilePath: VirtualPath.Handle? + if let apiDescriptorDirectory = apiDescriptorDirectory { + apiDescriptorFilePath = apiDescriptorDirectory + .appending(component: "\(moduleOutputInfo.name).\(target.moduleTriple.triple).swift.sdkdb") + .intern() + } else { + apiDescriptorFilePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .jsonAPIDescriptor, isOutputOptions: [], + outputPath: pathOptions.apiDescriptorPath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name) + } + + // --------------------- + // Swift interface paths + let swiftInterfacePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .swiftInterface, isOutputOptions: [.emitModuleInterface], + outputPath: pathOptions.moduleInterfacePath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name) + + let givenPrivateInterfacePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .privateSwiftInterface, isOutputOptions: [], + outputPath: pathOptions.privateInterfacePath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name) + let givenPackageInterfacePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .packageSwiftInterface, isOutputOptions: [], + outputPath: pathOptions.packageInterfacePath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: outputFileMap, + moduleName: moduleOutputInfo.name) + + // Always emit the private swift interface if a public interface is emitted. + // With the introduction of features like @_spi_available, we may print + // public and private interfaces differently even from the same codebase. + // For this reason, we should always print private interfaces so that we + // don’t mix the public interfaces with private Clang modules. + let swiftPrivateInterfacePath: VirtualPath.Handle? + if let privateInterfacePath = givenPrivateInterfacePath { + swiftPrivateInterfacePath = privateInterfacePath + } else if let swiftInterfacePath = swiftInterfacePath { + swiftPrivateInterfacePath = try VirtualPath.lookup(swiftInterfacePath) + .replacingExtension(with: .privateSwiftInterface).intern() + } else { + swiftPrivateInterfacePath = nil + } + + let swiftPackageInterfacePath: VirtualPath.Handle? + if let packageName = packageName, + !packageName.isEmpty { + // Generate a package interface if built with -package-name required for + // decls with the `package` access level. The `.package.swiftinterface` + // contains package decls as well as SPI and public decls (superset of a + // private interface). + if let givenPackageInterfacePath = givenPackageInterfacePath { + swiftPackageInterfacePath = givenPackageInterfacePath + } else if let swiftInterfacePath = swiftInterfacePath { + swiftPackageInterfacePath = try VirtualPath.lookup(swiftInterfacePath) + .replacingExtension(with: .packageSwiftInterface).intern() + } else { + swiftPackageInterfacePath = nil + } + } else { + swiftPackageInterfacePath = nil + } + + return SupplementalModuleTargetOutputPaths( + moduleDocOutputPath: moduleDocOutputPath, + swiftInterfacePath: swiftInterfacePath, + swiftPrivateInterfacePath: swiftPrivateInterfacePath, + swiftPackageInterfacePath: swiftPackageInterfacePath, + moduleSourceInfoPath: moduleSourceInfoPath, + apiDescriptorFilePath: apiDescriptorFilePath, + abiDescriptorFilePath: abiDescriptorFilePath) + } - /// Path to the Swift interface file. - let swiftInterfacePath: VirtualPath.Handle? + /// Structure storing paths to supplemental outputs for the target module + let moduleOutputPaths: SupplementalModuleTargetOutputPaths - /// Path to the Swift private interface file. - let swiftPrivateInterfacePath: VirtualPath.Handle? + /// Structure storing paths to supplemental outputs for the target variant + let variantModuleOutputPaths: SupplementalModuleTargetOutputPaths? /// File type for the optimization record. let optimizationRecordFileType: FileType? @@ -340,12 +614,10 @@ public struct Driver { /// Path to the optimization record. let optimizationRecordPath: VirtualPath.Handle? - /// Path to the Swift module source information file. - let moduleSourceInfoPath: VirtualPath.Handle? - /// Path to the module's digester baseline file. let digesterBaselinePath: VirtualPath.Handle? + /// The mode the API digester should run in. let digesterMode: DigesterMode @@ -357,6 +629,10 @@ public struct Driver { /// as explicit inputs by the various compilation jobs. @_spi(Testing) public var explicitDependencyBuildPlanner: ExplicitDependencyBuildPlanner? = nil + /// A reference to the instance of libSwiftScan which is shared with the driver's + /// `InterModuleDependencyOracle`, but also used for non-scanning tasks, such as target info + /// and supported compiler feature queries + @_spi(Testing) public var swiftScanLibInstance: SwiftScan? = nil /// An oracle for querying inter-module dependencies /// Can either be an argument to the driver in many-module contexts where dependency information /// is shared across many targets; otherwise, a new instance is created by the driver itself. @@ -381,6 +657,7 @@ public struct Driver { @_spi(Testing) public enum KnownCompilerFeature: String { case emit_abi_descriptor = "emit-abi-descriptor" + case compilation_caching = "compilation-caching" } lazy var sdkPath: VirtualPath? = { @@ -399,24 +676,6 @@ public struct Driver { .appending(component: "Frameworks") } () - lazy var abiDescriptorPath: TypedVirtualPath? = { - guard isFeatureSupported(.emit_abi_descriptor) else { - return nil - } - // Emit the descriptor only on platforms where Library Evolution is supported, - // or opted-into explicitly. - guard targetTriple.isDarwin || parsedOptions.hasArgument(.enableLibraryEvolution) else { - return nil - } - guard let moduleOutput = moduleOutputInfo.output else { - return nil - } - guard let path = try? VirtualPath.lookup(moduleOutput.outputPath).replacingExtension(with: .jsonABIBaseline) else { - return nil - } - return TypedVirtualPath(file: path.intern(), type: .jsonABIBaseline) - }() - public static func isOptionFound(_ opt: String, allOpts: Set) -> Bool { var current = opt while(true) { @@ -440,6 +699,18 @@ public struct Driver { return supportedFrontendFeatures.contains(feature.rawValue) } + public func getSwiftScanLibPath() throws -> AbsolutePath? { + return try toolchain.lookupSwiftScanLib() + } + + func findBlocklists() throws -> [AbsolutePath] { + if let mockBlocklistDir = env["_SWIFT_DRIVER_MOCK_BLOCK_LIST_DIR"] { + // Use testing block-list directory. + return try Driver.findBlocklists(RelativeTo: try AbsolutePath(validating: mockBlocklistDir)) + } + return try Driver.findBlocklists(RelativeTo: try toolchain.executableDir) + } + @_spi(Testing) public static func findBlocklists(RelativeTo execDir: AbsolutePath) throws -> [AbsolutePath] { // Expect to find all blocklists in such dir: @@ -475,23 +746,23 @@ public struct Driver { stdErrQueue.sync { let stream = stderrStream if !(diagnostic.location is UnknownLocation) { - stream <<< diagnostic.location.description <<< ": " + stream.send("\(diagnostic.location.description): ") } switch diagnostic.message.behavior { case .error: - stream <<< "error: " + stream.send("error: ") case .warning: - stream <<< "warning: " + stream.send("warning: ") case .note: - stream <<< "note: " + stream.send("note: ") case .remark: - stream <<< "remark: " + stream.send("remark: ") case .ignored: break } - stream <<< diagnostic.localizedDescription <<< "\n" + stream.send("\(diagnostic.localizedDescription)\n") stream.flush() } } @@ -515,6 +786,33 @@ public struct Driver { fileSystem: fileSystem, executor: executor, integratedDriver: integratedDriver, + compilerIntegratedTooling: false, + compilerExecutableDir: compilerExecutableDir, + externalTargetModuleDetailsMap: externalTargetModuleDetailsMap, + interModuleDependencyOracle: interModuleDependencyOracle + ) + } + + @available(*, deprecated, renamed: "init(args:env:diagnosticsOutput:fileSystem:executor:integratedDriver:compilerIntegratedTooling:compilerExecutableDir:externalTargetModuleDetailsMap:interModuleDependencyOracle:)") + public init( + args: [String], + env: [String: String] = ProcessEnv.vars, + diagnosticsOutput: DiagnosticsOutput = .engine(DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler])), + fileSystem: FileSystem = localFileSystem, + executor: DriverExecutor, + integratedDriver: Bool = true, + compilerExecutableDir: AbsolutePath? = nil, + externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil, + interModuleDependencyOracle: InterModuleDependencyOracle? = nil + ) throws { + try self.init( + args: args, + env: env, + diagnosticsOutput: diagnosticsOutput, + fileSystem: fileSystem, + executor: executor, + integratedDriver: integratedDriver, + compilerIntegratedTooling: false, compilerExecutableDir: compilerExecutableDir, externalTargetModuleDetailsMap: externalTargetModuleDetailsMap, interModuleDependencyOracle: interModuleDependencyOracle @@ -535,6 +833,8 @@ public struct Driver { /// is present to streamline testing, it shouldn't be used in production. /// - Parameter integratedDriver: Used to distinguish whether the driver is being used as /// an executable or as a library. + /// - Parameter compilerIntegratedTooling: If true, this code is executed in the context of a + /// Swift compiler image which contains symbols normally queried from a libSwiftScan instance. /// - Parameter compilerExecutableDir: Directory that contains the compiler executable to be used. /// Used when in `integratedDriver` mode as a substitute for the driver knowing its executable path. /// - Parameter externalTargetModuleDetailsMap: A dictionary of external targets that are a part of @@ -549,6 +849,7 @@ public struct Driver { fileSystem: FileSystem = localFileSystem, executor: DriverExecutor, integratedDriver: Bool = true, + compilerIntegratedTooling: Bool = false, compilerExecutableDir: AbsolutePath? = nil, externalTargetModuleDetailsMap: ExternalTargetModuleDetailsMap? = nil, interModuleDependencyOracle: InterModuleDependencyOracle? = nil @@ -556,6 +857,7 @@ public struct Driver { self.env = env self.fileSystem = fileSystem self.integratedDriver = integratedDriver + self.compilerIntegratedTooling = compilerIntegratedTooling let diagnosticsEngine: DiagnosticsEngine switch diagnosticsOutput { @@ -591,19 +893,15 @@ public struct Driver { diagnosticEngine: diagnosticsEngine, compilerMode: compilerMode) - let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING") - self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride - self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE") - // Compute the working directory. - workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in + self.workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in let cwd = fileSystem.currentWorkingDirectory return try cwd.map{ try AbsolutePath(validating: workingDirectoryArg.asSingle, relativeTo: $0) } ?? AbsolutePath(validating: workingDirectoryArg.asSingle) } - // Apply the working directory to the parsed options. - if let workingDirectory = self.workingDirectory { - try Self.applyWorkingDirectory(workingDirectory, to: &self.parsedOptions) + if let specifiedWorkingDir = self.workingDirectory { + // Apply the working directory to the parsed options if passed explicitly. + try Self.applyWorkingDirectory(specifiedWorkingDir, to: &self.parsedOptions) } let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, @@ -615,7 +913,7 @@ public struct Driver { self.useStaticResourceDir = staticExecutable || staticStdlib // Build the toolchain and determine target information. - (self.toolchain, self.frontendTargetInfo, self.swiftCompilerPrefixArgs) = + (self.toolchain, self.swiftCompilerPrefixArgs) = try Self.computeToolchain( &self.parsedOptions, diagnosticsEngine: diagnosticEngine, compilerMode: self.compilerMode, env: env, @@ -624,13 +922,40 @@ public struct Driver { workingDirectory: self.workingDirectory, compilerExecutableDir: compilerExecutableDir) + // Create an instance of an inter-module dependency oracle, if the driver's + // client did not provide one. The clients are expected to provide an oracle + // when they wish to share module dependency information across targets. + if let dependencyOracle = interModuleDependencyOracle { + self.interModuleDependencyOracle = dependencyOracle + } else { + self.interModuleDependencyOracle = InterModuleDependencyOracle() + } + + self.swiftScanLibInstance = try Self.initializeSwiftScanInstance(&parsedOptions, + diagnosticsEngine: diagnosticEngine, + toolchain: self.toolchain, + interModuleDependencyOracle: self.interModuleDependencyOracle, + fileSystem: self.fileSystem, + compilerIntegratedTooling: self.compilerIntegratedTooling) + // Compute the host machine's triple self.hostTriple = try Self.computeHostTriple(&self.parsedOptions, diagnosticsEngine: diagnosticEngine, + libSwiftScan: self.swiftScanLibInstance, toolchain: self.toolchain, executor: self.executor, fileSystem: fileSystem, - workingDirectory: self.workingDirectory, - swiftCompilerPrefixArgs: self.swiftCompilerPrefixArgs) + workingDirectory: self.workingDirectory) + + // Compute the entire target info, including runtime resource paths + self.frontendTargetInfo = try Self.computeTargetInfo(&self.parsedOptions, diagnosticsEngine: diagnosticEngine, + compilerMode: self.compilerMode, env: env, + executor: self.executor, + libSwiftScan: self.swiftScanLibInstance, + toolchain: self.toolchain, + fileSystem: fileSystem, + useStaticResourceDir: self.useStaticResourceDir, + workingDirectory: self.workingDirectory, + compilerExecutableDir: compilerExecutableDir) // Classify and collect all of the input files. let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem) @@ -657,35 +982,21 @@ public struct Driver { } if let workingDirectory = self.workingDirectory { + // Input paths are prefixed with the working directory when specified, + // apply the same logic to the output file map keys. self.outputFileMap = outputFileMap?.resolveRelativePaths(relativeTo: workingDirectory) } else { self.outputFileMap = outputFileMap } } - // Create an instance of an inter-module dependency oracle, if the driver's - // client did not provide one. The clients are expected to provide an oracle - // when they wish to share module dependency information across targets. - if let dependencyOracle = interModuleDependencyOracle { - self.interModuleDependencyOracle = dependencyOracle - } else { - self.interModuleDependencyOracle = InterModuleDependencyOracle() - } - self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine) self.shouldUseInputFileList = inputFiles.count > fileListThreshold - if shouldUseInputFileList { - let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation) - self.allSourcesFileList = VirtualPath.createUniqueFilelist(RelativePath("sources"), - .list(swiftInputs.map(\.file))) - } else { - self.allSourcesFileList = nil - } self.lto = Self.ltoKind(&parsedOptions, diagnosticsEngine: diagnosticsEngine) // Figure out the primary outputs from the driver. - (self.compilerOutputType, self.linkerOutputType) = - Self.determinePrimaryOutputs(&parsedOptions, targetTriple: self.frontendTargetInfo.target.triple, + (self.compilerOutputType, self.linkerOutputType) = + Self.determinePrimaryOutputs(&parsedOptions, targetTriple: self.frontendTargetInfo.target.triple, driverKind: driverKind, diagnosticsEngine: diagnosticEngine) // Multithreading. @@ -722,7 +1033,10 @@ public struct Driver { diagnosticsEngine: diagnosticEngine) // Compute debug information output. - self.debugInfo = Self.computeDebugInfo(&parsedOptions, diagnosticsEngine: diagnosticEngine) + let defaultDwarfVersion = self.toolchain.getDefaultDwarfVersion(targetTriple: self.frontendTargetInfo.target.triple) + self.debugInfo = Self.computeDebugInfo(&parsedOptions, + defaultDwarfVersion: defaultDwarfVersion, + diagnosticsEngine: diagnosticEngine) // Error if package-name is passed but the input is empty; if // package-name is not passed but `package` decls exist, error @@ -734,10 +1048,24 @@ public struct Driver { // Determine the module we're building and whether/how the module file itself will be emitted. self.moduleOutputInfo = try Self.computeModuleInfo( - &parsedOptions, compilerOutputType: compilerOutputType, compilerMode: compilerMode, linkerOutputType: linkerOutputType, - debugInfoLevel: debugInfo.level, diagnosticsEngine: diagnosticEngine, + &parsedOptions, + modulePath: parsedOptions.getLastArgument(.emitModulePath)?.asSingle, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + linkerOutputType: linkerOutputType, + debugInfoLevel: debugInfo.level, + diagnosticsEngine: diagnosticEngine, workingDirectory: self.workingDirectory) + self.variantModuleOutputInfo = try Self.computeVariantModuleInfo( + &parsedOptions, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + linkerOutputType: linkerOutputType, + debugInfoLevel: debugInfo.level, + diagnosticsEngine: diagnosticsEngine, + workingDirectory: workingDirectory) + // Should we schedule a separate emit-module job? self.emitModuleSeparately = Self.computeEmitModuleSeparately(parsedOptions: &parsedOptions, compilerMode: compilerMode, @@ -757,14 +1085,9 @@ public struct Driver { parsedOptions: parsedOptions, recordedInputModificationDates: recordedInputModificationDates) - self.importedObjCHeader = try Self.computeImportedObjCHeader(&parsedOptions, compilerMode: compilerMode, diagnosticEngine: diagnosticEngine) - self.bridgingPrecompiledHeader = try Self.computeBridgingPrecompiledHeader(&parsedOptions, - compilerMode: compilerMode, - importedObjCHeader: importedObjCHeader, - outputFileMap: outputFileMap) - self.supportedFrontendFlags = try Self.computeSupportedCompilerArgs(of: self.toolchain, + libSwiftScan: self.swiftScanLibInstance, parsedOptions: &self.parsedOptions, diagnosticsEngine: diagnosticEngine, fileSystem: fileSystem, @@ -779,6 +1102,83 @@ public struct Driver { } self.supportedFrontendFeatures = try Self.computeSupportedCompilerFeatures(of: self.toolchain, env: env) + // Caching options. + let cachingEnabled = parsedOptions.hasArgument(.cacheCompileJob) || env.keys.contains("SWIFT_ENABLE_CACHING") + if cachingEnabled { + if !parsedOptions.hasArgument(.driverExplicitModuleBuild) { + diagnosticsEngine.emit(.warning("-cache-compile-job cannot be used without explicit module build, turn off caching"), + location: nil) + self.enableCaching = false + } else { + self.enableCaching = true + } + } else { + self.enableCaching = false + } + + // PCH related options. + if parsedOptions.hasArgument(.importObjcHeader) { + // Check for conflicting options. + if parsedOptions.hasArgument(.importUnderlyingModule) { + diagnosticEngine.emit(.error_framework_bridging_header) + } + + if parsedOptions.hasArgument(.emitModuleInterface, .emitModuleInterfacePath) { + diagnosticEngine.emit(.error_bridging_header_module_interface) + } + } + var maybeNeedPCH = parsedOptions.hasFlag(positive: .enableBridgingPch, negative: .disableBridgingPch, default: true) + if enableCaching && !maybeNeedPCH { + diagnosticsEngine.emit(.warning("-disable-bridging-pch is ignored because compilation caching (-cache-compile-job) is used"), + location: nil) + maybeNeedPCH = true + } + self.producePCHJob = maybeNeedPCH + + if let objcHeaderPathArg = parsedOptions.getLastArgument(.importObjcHeader) { + self.originalObjCHeaderFile = try? VirtualPath.intern(path: objcHeaderPathArg.asSingle) + } else { + self.originalObjCHeaderFile = nil + } + + if parsedOptions.hasFlag(positive: .autoBridgingHeaderChaining, + negative: .noAutoBridgingHeaderChaining, + default: false) || cachingEnabled { + if producePCHJob { + self.bridgingHeaderChaining = true + } else { + diagnosticEngine.emit(.warning("-auto-bridging-header-chaining requires generatePCH job, no chaining will be performed")) + self.bridgingHeaderChaining = false + } + } else { + self.bridgingHeaderChaining = false + } + + self.useClangIncludeTree = !parsedOptions.hasArgument(.noClangIncludeTree) && !env.keys.contains("SWIFT_CACHING_USE_CLANG_CAS_FS") + self.scannerPrefixMap = try Self.computeScanningPrefixMapper(&parsedOptions) + if let sdkMapping = parsedOptions.getLastArgument(.scannerPrefixMapSdk)?.asSingle { + self.scannerPrefixMapSDK = try AbsolutePath(validating: sdkMapping) + } else { + self.scannerPrefixMapSDK = nil + } + if let toolchainMapping = parsedOptions.getLastArgument(.scannerPrefixMapToolchain)?.asSingle { + self.scannerPrefixMapToolchain = try AbsolutePath(validating: toolchainMapping) + } else { + self.scannerPrefixMapToolchain = nil + } + + // Initialize the CAS instance + if self.swiftScanLibInstance != nil && + self.enableCaching && + self.supportedFrontendFeatures.contains(KnownCompilerFeature.compilation_caching.rawValue) { + self.cas = + try self.interModuleDependencyOracle.getOrCreateCAS(pluginPath: try Self.getCASPluginPath(parsedOptions: &self.parsedOptions, + toolchain: self.toolchain), + onDiskPath: try Self.getOnDiskCASPath(parsedOptions: &self.parsedOptions, + toolchain: self.toolchain), + pluginOptions: try Self.getCASPluginOptions(parsedOptions: &self.parsedOptions)) + } + self.enabledSanitizers = try Self.parseSanitizerArgValues( &parsedOptions, diagnosticEngine: diagnosticEngine, @@ -787,6 +1187,8 @@ public struct Driver { Self.validateSanitizerAddressUseOdrIndicatorFlag(&parsedOptions, diagnosticEngine: diagnosticsEngine, addressSanitizerEnabled: enabledSanitizers.contains(.address)) + Self.validateSanitizeStableABI(&parsedOptions, diagnosticEngine: diagnosticsEngine, addressSanitizerEnabled: enabledSanitizers.contains(.address)) + Self.validateSanitizerRecoverArgValues(&parsedOptions, diagnosticEngine: diagnosticsEngine, enabledSanitizers: enabledSanitizers) Self.validateSanitizerCoverageArgs(&parsedOptions, @@ -873,24 +1275,53 @@ public struct Driver { emitModuleSeparately: emitModuleSeparately, outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) - self.moduleDocOutputPath = try Self.computeModuleDocOutputPath( - &parsedOptions, moduleOutputPath: self.moduleOutputInfo.output?.outputPath, - compilerOutputType: compilerOutputType, - compilerMode: compilerMode, - outputFileMap: self.outputFileMap, - moduleName: moduleOutputInfo.name) let projectDirectory = Self.computeProjectDirectoryPath( - moduleOutputPath: self.moduleOutputInfo.output?.outputPath, + moduleOutputPath: moduleOutputInfo.output?.outputPath, fileSystem: self.fileSystem) - self.moduleSourceInfoPath = try Self.computeModuleSourceInfoOutputPath( + + var apiDescriptorDirectory: VirtualPath? = nil + if let apiDescriptorDirectoryEnvVar = env["TAPI_SDKDB_OUTPUT_PATH"] { + apiDescriptorDirectory = try VirtualPath(path: apiDescriptorDirectoryEnvVar) + } else if let ldTraceFileEnvVar = env["LD_TRACE_FILE"] { + apiDescriptorDirectory = try VirtualPath(path: ldTraceFileEnvVar).parentDirectory.appending(component: "SDKDB") + } + + self.moduleOutputPaths = try Self.computeModuleOutputPaths( + &parsedOptions, + moduleName: moduleOutputInfo.name, + packageName: self.packageName, + moduleOutputInfo: self.moduleOutputInfo, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + emitModuleSeparately: emitModuleSeparately, + outputFileMap: self.outputFileMap, + projectDirectory: projectDirectory, + apiDescriptorDirectory: apiDescriptorDirectory, + supportedFrontendFeatures: self.supportedFrontendFeatures, + target: frontendTargetInfo.target, + isVariant: false) + + if let variantModuleOutputInfo = self.variantModuleOutputInfo, + let targetVariant = self.frontendTargetInfo.targetVariant { + self.variantModuleOutputPaths = try Self.computeModuleOutputPaths( &parsedOptions, - moduleOutputPath: self.moduleOutputInfo.output?.outputPath, + moduleName: variantModuleOutputInfo.name, + packageName: self.packageName, + moduleOutputInfo: variantModuleOutputInfo, compilerOutputType: compilerOutputType, compilerMode: compilerMode, + emitModuleSeparately: true, // variant module is always independent outputFileMap: self.outputFileMap, - moduleName: moduleOutputInfo.name, - projectDirectory: projectDirectory) + projectDirectory: projectDirectory, + apiDescriptorDirectory: apiDescriptorDirectory, + supportedFrontendFeatures: self.supportedFrontendFeatures, + target: targetVariant, + isVariant: true) + } else { + self.variantModuleOutputPaths = nil + } + self.digesterBaselinePath = try Self.computeDigesterBaselineOutputPath( &parsedOptions, moduleOutputPath: self.moduleOutputInfo.output?.outputPath, @@ -900,35 +1331,6 @@ public struct Driver { outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name, projectDirectory: projectDirectory) - self.swiftInterfacePath = try Self.computeSupplementaryOutputPath( - &parsedOptions, type: .swiftInterface, isOutputOptions: [.emitModuleInterface], - outputPath: .emitModuleInterfacePath, - compilerOutputType: compilerOutputType, - compilerMode: compilerMode, - emitModuleSeparately: emitModuleSeparately, - outputFileMap: self.outputFileMap, - moduleName: moduleOutputInfo.name) - let givenPrivateInterfacePath = try Self.computeSupplementaryOutputPath( - &parsedOptions, type: .privateSwiftInterface, isOutputOptions: [], - outputPath: .emitPrivateModuleInterfacePath, - compilerOutputType: compilerOutputType, - compilerMode: compilerMode, - emitModuleSeparately: emitModuleSeparately, - outputFileMap: self.outputFileMap, - moduleName: moduleOutputInfo.name) - - // Always emitting private swift interfaces if public interfaces are emitted.' - // With the introduction of features like @_spi_available, we may print public - // and private interfaces differently even from the same codebase. For this reason, - // we should always print private interfaces so that we don’t mix the public interfaces - // with private Clang modules. - if let swiftInterfacePath = self.swiftInterfacePath, - givenPrivateInterfacePath == nil { - self.swiftPrivateInterfacePath = try VirtualPath.lookup(swiftInterfacePath) - .replacingExtension(with: .privateSwiftInterface).intern() - } else { - self.swiftPrivateInterfacePath = givenPrivateInterfacePath - } var optimizationRecordFileType = FileType.yamlOptimizationRecord if let argument = parsedOptions.getLastArgument(.saveOptimizationRecordEQ)?.asSingle { @@ -956,15 +1358,16 @@ public struct Driver { Self.validateDigesterArgs(&parsedOptions, moduleOutputInfo: moduleOutputInfo, digesterMode: self.digesterMode, - swiftInterfacePath: self.swiftInterfacePath, + swiftInterfacePath: self.moduleOutputPaths.swiftInterfacePath, diagnosticEngine: diagnosticsEngine) try verifyOutputOptions() } public mutating func planBuild() throws -> [Job] { - let (jobs, incrementalCompilationState) = try planPossiblyIncrementalBuild() + let (jobs, incrementalCompilationState, intermoduleDependencyGraph) = try planPossiblyIncrementalBuild() self.incrementalCompilationState = incrementalCompilationState + self.intermoduleDependencyGraph = intermoduleDependencyGraph return jobs } } @@ -1352,6 +1755,38 @@ extension Diagnostic.Message { } } +extension Driver { + func explainModuleDependency(_ explainModuleName: String, allPaths: Bool) throws { + guard let dependencyPlanner = explicitDependencyBuildPlanner else { + fatalError("Cannot explain dependency without Explicit Build Planner") + } + guard let dependencyPaths = try dependencyPlanner.explainDependency(explainModuleName, allPaths: allPaths) else { + diagnosticEngine.emit(.remark("No such module dependency found: '\(explainModuleName)'")) + return + } + diagnosticEngine.emit(.remark("Module '\(moduleOutputInfo.name)' depends on '\(explainModuleName)'")) + for path in dependencyPaths { + var pathString:String = "" + for (index, moduleId) in path.enumerated() { + switch moduleId { + case .swift(let moduleName): + pathString = pathString + "[" + moduleName + "]" + case .swiftPrebuiltExternal(let moduleName): + pathString = pathString + "[" + moduleName + "]" + case .clang(let moduleName): + pathString = pathString + "[" + moduleName + "](ObjC)" + case .swiftPlaceholder(_): + fatalError("Unexpected unresolved Placeholder module") + } + if index < path.count - 1 { + pathString = pathString + " -> " + } + } + diagnosticEngine.emit(.note(pathString)) + } + } +} + extension Driver { /// Determine the driver kind based on the command-line arguments, consuming the arguments /// conveying this information. @@ -1398,39 +1833,15 @@ extension Driver { } // If we're only supposed to explain a dependency on a given module, do so now. - if let explainModuleName = parsedOptions.getLastArgument(.explainModuleDependency) { - guard let dependencyPlanner = explicitDependencyBuildPlanner else { - fatalError("Cannot explain dependency without Explicit Build Planner") - } - guard let dependencyPaths = try dependencyPlanner.explainDependency(explainModuleName.asSingle) else { - diagnosticEngine.emit(.remark("No such module dependency found: '\(explainModuleName.asSingle)'")) - return - } - diagnosticEngine.emit(.remark("Module '\(moduleOutputInfo.name)' depends on '\(explainModuleName.asSingle)'")) - for path in dependencyPaths { - var pathString:String = "" - for (index, moduleId) in path.enumerated() { - switch moduleId { - case .swift(let moduleName): - pathString = pathString + "[" + moduleName + "]" - case .swiftPrebuiltExternal(let moduleName): - pathString = pathString + "[" + moduleName + "]" - case .clang(let moduleName): - pathString = pathString + "[" + moduleName + "](ObjC)" - case .swiftPlaceholder(_): - fatalError("Unexpected unresolved Placeholder module") - } - if index < path.count - 1 { - pathString = pathString + " -> " - } - } - diagnosticEngine.emit(.note(pathString)) - } + if let explainModuleName = parsedOptions.getLastArgument(.explainModuleDependencyDetailed) { + try explainModuleDependency(explainModuleName.asSingle, allPaths: true) + } else if let explainModuleNameDetailed = parsedOptions.getLastArgument(.explainModuleDependency) { + try explainModuleDependency(explainModuleNameDetailed.asSingle, allPaths: false) } if parsedOptions.contains(.driverPrintOutputFileMap) { if let outputFileMap = self.outputFileMap { - stderrStream <<< outputFileMap.description + stderrStream.send(outputFileMap.description) stderrStream.flush() } else { diagnosticEngine.emit(.error_no_output_file_map_specified) @@ -1507,9 +1918,9 @@ extension Driver { // Print the driver source version first before we print the compiler // versions. if inPlaceJob.kind == .versionRequest && !Driver.driverSourceVersion.isEmpty { - stderrStream <<< "swift-driver version: " <<< Driver.driverSourceVersion <<< " " + stderrStream.send("swift-driver version: \(Driver.driverSourceVersion) ") if let blocklistVersion = try Driver.findCompilerClientsConfigVersion(RelativeTo: try toolchain.executableDir) { - stderrStream <<< blocklistVersion <<< " " + stderrStream.send("\(blocklistVersion) ") } stderrStream.flush() } @@ -1517,7 +1928,7 @@ extension Driver { if parsedOptions.contains(.v) { let arguments: [String] = try executor.resolver.resolveArgumentList(for: inPlaceJob, useResponseFiles: forceResponseFiles ? .forced : .heuristic) - stdoutStream <<< arguments.map { $0.spm_shellEscaped() }.joined(separator: " ") <<< "\n" + stdoutStream.send("\(arguments.map { $0.spm_shellEscaped() }.joined(separator: " "))\n") stdoutStream.flush() } try executor.execute(job: inPlaceJob, @@ -1574,10 +1985,7 @@ extension Driver { // In case the write fails, don't crash the build. // A mitigation to rdar://76359678. // If the write fails, import incrementality is lost, but it is not a fatal error. - guard - let buildRecordInfo = self.buildRecordInfo, - let absPath = buildRecordInfo.buildRecordPath.absolutePath - else { + guard let buildRecordInfo = self.buildRecordInfo, let incrementalCompilationState = self.incrementalCompilationState else { return } @@ -1586,71 +1994,28 @@ extension Driver { $0.skippedCompilationInputs }) - if - let incrementalCompilationState = self.incrementalCompilationState, - incrementalCompilationState.info.isCrossModuleIncrementalBuildEnabled - { - do { - try incrementalCompilationState.writeDependencyGraph(to: buildRecordInfo.dependencyGraphPath, buildRecord) - } catch { - diagnosticEngine.emit( - .warning("next compile won't be incremental; could not write dependency graph: \(error.localizedDescription)")) - /// Ensure that a bogus dependency graph is not used next time. - buildRecordInfo.removeBuildRecord() - buildRecordInfo.removeInterModuleDependencyGraph() - return - } - do { - try incrementalCompilationState.writeInterModuleDependencyGraph(buildRecordInfo) - } catch { - diagnosticEngine.emit( - .warning("next compile must run a full dependency scan; could not write inter-module dependency graph: \(error.localizedDescription)")) - buildRecordInfo.removeBuildRecord() - buildRecordInfo.removeInterModuleDependencyGraph() - return - } - } else { - // FIXME: This is all legacy code. Once the cross module incremental build - // becomes the default: - // - // 1) Delete this branch - // 2) Delete the parts of the incremental build that talk about anything - // derived from `buildRecordPath` - // 3) Delete the Yams dependency. - - // Before writing to the dependencies file path, preserve any previous file - // that may have been there. No error handling -- this is just a nicety, it - // doesn't matter if it fails. - // Added for the sake of compatibility with the legacy driver. - try? fileSystem.move( - from: absPath, to: absPath.appending(component: absPath.basename + "~")) - - guard let contents = buildRecord.encode(diagnosticEngine: diagnosticEngine) else { - diagnosticEngine.emit(.warning_could_not_write_build_record(absPath)) - return - } - - do { - try fileSystem.writeFileContents(absPath, - bytes: ByteString(encodingAsUTF8: contents)) - } catch { - diagnosticEngine.emit(.warning_could_not_write_build_record(absPath)) - } + do { + try incrementalCompilationState.writeDependencyGraph(to: buildRecordInfo.dependencyGraphPath, buildRecord) + } catch { + diagnosticEngine.emit( + .warning("next compile won't be incremental; could not write dependency graph: \(error.localizedDescription)")) + /// Ensure that a bogus dependency graph is not used next time. + buildRecordInfo.removeBuildRecord() + return } } private func printBindings(_ job: Job) { - stdoutStream <<< #"# ""# <<< targetTriple.triple - stdoutStream <<< #"" - ""# <<< job.tool.basename - stdoutStream <<< #"", inputs: ["# - stdoutStream <<< job.displayInputs.map { "\"" + $0.file.name + "\"" }.joined(separator: ", ") + stdoutStream.send(#"# ""#).send(targetTriple.triple) + stdoutStream.send(#"" - ""#).send(job.tool.basename) + stdoutStream.send(#"", inputs: ["#) + stdoutStream.send(job.displayInputs.map { "\"" + $0.file.name + "\"" }.joined(separator: ", ")) - stdoutStream <<< "], output: {" + stdoutStream.send("], output: {") - stdoutStream <<< job.outputs.map { $0.type.name + ": \"" + $0.file.name + "\"" }.joined(separator: ", ") + stdoutStream.send(job.outputs.map { $0.type.name + ": \"" + $0.file.name + "\"" }.joined(separator: ", ")) - stdoutStream <<< "}" - stdoutStream <<< "\n" + stdoutStream.send("}\n") stdoutStream.flush() } @@ -1659,7 +2024,7 @@ extension Driver { /// The swift-driver doesn't have actions, so the logic here takes the jobs and tries /// to mimic the actions that would be created by the C++ driver and /// prints them in *hopefully* the same order. - private func printActions(_ jobs: [Job]) { + private mutating func printActions(_ jobs: [Job]) { defer { stdoutStream.flush() } @@ -1721,19 +2086,19 @@ extension Driver { } // Print current Job - stdoutStream <<< nextId <<< ": " <<< job.kind.rawValue <<< ", {" + stdoutStream.send("\(nextId): ").send(job.kind.rawValue).send(", {") switch job.kind { // Don't sort for compile jobs. Puts pch last case .compile: - stdoutStream <<< inputIds.map(\.description).joined(separator: ", ") + stdoutStream.send(inputIds.map(\.description).joined(separator: ", ")) default: - stdoutStream <<< inputIds.sorted().map(\.description).joined(separator: ", ") + stdoutStream.send(inputIds.sorted().map(\.description).joined(separator: ", ")) } var typeName = job.outputs.first?.type.name if typeName == nil { typeName = "none" } - stdoutStream <<< "}, " <<< typeName! <<< "\n" + stdoutStream.send("}, \(typeName!)\n") jobIdMap[job] = nextId nextId += 1 } @@ -1741,16 +2106,16 @@ extension Driver { private static func printInputIfNew(_ input: TypedVirtualPath, inputIdMap: inout [TypedVirtualPath: UInt], nextId: inout UInt) { if inputIdMap[input] == nil { - stdoutStream <<< nextId <<< ": " <<< "input, " - stdoutStream <<< "\"" <<< input.file <<< "\", " <<< input.type <<< "\n" + stdoutStream.send("\(nextId): input, ") + stdoutStream.send("\"\(input.file)\", \(input.type)\n") inputIdMap[input] = nextId nextId += 1 } } private func printVersion(outputStream: inout S) throws { - outputStream <<< frontendTargetInfo.compilerVersion <<< "\n" - outputStream <<< "Target: \(frontendTargetInfo.target.triple.triple)\n" + outputStream.send("\(frontendTargetInfo.compilerVersion)\n") + outputStream.send("Target: \(frontendTargetInfo.target.triple.triple)\n") outputStream.flush() } } @@ -2007,9 +2372,9 @@ extension Driver { let filePath = VirtualPath.absolute(absPath.appending(component: "main.swift")) try fileSystem.writeFileContents(filePath) { file in - file <<< ###"#sourceLocation(file: "-e", line: 1)\###n"### + file.send(###"#sourceLocation(file: "-e", line: 1)\###n"###) for option in parsedOptions.arguments(for: .e) { - file <<< option.argument.asSingle <<< "\n" + file.send("\(option.argument.asSingle)\n") } } @@ -2063,7 +2428,10 @@ extension Driver { case .emitSibgen: compilerOutputType = .raw_sib - case .emitIrgen, .emitIr: + case .emitIrgen: + compilerOutputType = .raw_llvmIr + + case .emitIr: compilerOutputType = .llvmIR case .emitBc: @@ -2084,8 +2452,8 @@ extension Driver { case .indexFile: compilerOutputType = .indexData - case .parse, .resolveImports, .typecheck, .experimentalLazyTypecheck, - .dumpParse, .printAst, .dumpTypeRefinementContexts, .dumpScopeMaps, + case .parse, .resolveImports, .typecheck, + .dumpParse, .printAst, .dumpAvailabilityScopes, .dumpScopeMaps, .dumpInterfaceHash, .dumpTypeInfo, .verifyDebugInfo: compilerOutputType = nil @@ -2111,8 +2479,8 @@ extension Driver { linkerOutputType = .executable } - // warn if -embed-bitcode is set and the output type is not an object - if parsedOptions.hasArgument(.embedBitcode) && compilerOutputType != .object { + // warn if -embed-bitcode is set + if parsedOptions.hasArgument(.embedBitcode) { diagnosticsEngine.emit(.warn_ignore_embed_bitcode) parsedOptions.eraseArgument(.embedBitcode) } @@ -2136,7 +2504,7 @@ extension Diagnostic.Message { } static var warn_ignore_embed_bitcode: Diagnostic.Message { - .warning("ignoring -embed-bitcode since no object file is being generated") + .warning("'-embed-bitcode' has been deprecated") } static var warn_ignore_embed_bitcode_marker: Diagnostic.Message { @@ -2228,7 +2596,9 @@ extension Diagnostic.Message { // Debug information extension Driver { /// Compute the level of debug information we are supposed to produce. - private static func computeDebugInfo(_ parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine) -> DebugInfo { + private static func computeDebugInfo(_ parsedOptions: inout ParsedOptions, + defaultDwarfVersion : UInt8, + diagnosticsEngine: DiagnosticsEngine) -> DebugInfo { var shouldVerify = parsedOptions.hasArgument(.verifyDebugInfo) for debugPrefixMap in parsedOptions.arguments(for: .debugPrefixMap) { @@ -2296,7 +2666,17 @@ extension Driver { diagnosticsEngine.emit(.error_argument_not_allowed_with(arg: fullNotAllowedOption, other: levelOption.spelling)) } - return DebugInfo(format: format, level: level, shouldVerify: shouldVerify) + // Determine the DWARF version. + var dwarfVersion: UInt8 = defaultDwarfVersion + if let versionArg = parsedOptions.getLastArgument(.dwarfVersion) { + if let parsedVersion = UInt8(versionArg.asSingle), parsedVersion >= 2 && parsedVersion <= 5 { + dwarfVersion = parsedVersion + } else { + diagnosticsEngine.emit(.error_invalid_arg_value(arg: .dwarfVersion, value: versionArg.asSingle)) + } + } + + return DebugInfo(format: format, dwarfVersion: dwarfVersion, level: level, shouldVerify: shouldVerify) } /// Parses the set of `-sanitize={sanitizer}` arguments and returns all the @@ -2329,14 +2709,15 @@ extension Driver { continue } + let stableAbi = sanitizer == .address && parsedOptions.contains(.sanitizeStableAbiEQ) // Support is determined by existence of the sanitizer library. // FIXME: Should we do this? This prevents cross-compiling with sanitizers // enabled. var sanitizerSupported = try toolchain.runtimeLibraryExists( - for: sanitizer, + for: stableAbi ? .address_stable_abi : sanitizer, targetInfo: targetInfo, parsedOptions: &parsedOptions, - isShared: sanitizer != .fuzzer + isShared: sanitizer != .fuzzer && !stableAbi ) if sanitizer == .thread { @@ -2370,7 +2751,7 @@ extension Driver { } // Check that we're one of the known supported targets for sanitizers. - if !(targetTriple.isWindows || targetTriple.isDarwin || targetTriple.os == .linux) { + if !(targetTriple.isWindows || targetTriple.isDarwin || targetTriple.os == .linux || targetTriple.os == .wasi) { diagnosticEngine.emit( .error_unsupported_opt_for_target( arg: "-sanitize=", @@ -2441,9 +2822,36 @@ extension Driver { return "" } + private static func computeVariantModuleInfo( + _ parsedOptions: inout ParsedOptions, + compilerOutputType: FileType?, + compilerMode: CompilerMode, + linkerOutputType: LinkOutputType?, + debugInfoLevel: DebugInfo.Level?, + diagnosticsEngine: DiagnosticsEngine, + workingDirectory: AbsolutePath? + ) throws -> ModuleOutputInfo? { + // If there is no target variant, then there is no target variant module. + // If there is no emit-variant-module, then there is not target variant + // module. + guard let variantModulePath = parsedOptions.getLastArgument(.emitVariantModulePath), + parsedOptions.hasArgument(.targetVariant) else { + return nil + } + return try computeModuleInfo(&parsedOptions, + modulePath: variantModulePath.asSingle, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + linkerOutputType: linkerOutputType, + debugInfoLevel: debugInfoLevel, + diagnosticsEngine: diagnosticsEngine, + workingDirectory: workingDirectory) + } + /// Determine how the module will be emitted and the name of the module. private static func computeModuleInfo( _ parsedOptions: inout ParsedOptions, + modulePath: String?, compilerOutputType: FileType?, compilerMode: CompilerMode, linkerOutputType: LinkOutputType?, @@ -2458,7 +2866,7 @@ extension Driver { } var moduleOutputKind: ModuleOutputKind? - if parsedOptions.hasArgument(.emitModule, .emitModulePath) { + if parsedOptions.hasArgument(.emitModule) || modulePath != nil { // The user has requested a module, so generate one and treat it as // top-level output. moduleOutputKind = .topLevel @@ -2468,7 +2876,7 @@ extension Driver { moduleOutputKind = .auxiliary } else if parsedOptions.hasArgument(.emitObjcHeader, .emitObjcHeaderPath, .emitModuleInterface, .emitModuleInterfacePath, - .emitPrivateModuleInterfacePath) { + .emitPrivateModuleInterfacePath, .emitPackageModuleInterfacePath) { // An option has been passed which requires whole-module knowledge, but we // don't have that. Generate a module, but treat it as an intermediate // output. @@ -2540,9 +2948,9 @@ extension Driver { // FIXME: Look in the output file map. It looks like it is weirdly // anchored to the first input? - if let modulePathArg = parsedOptions.getLastArgument(.emitModulePath) { + if let modulePathArg = modulePath { // The module path was specified. - moduleOutputPath = try VirtualPath(path: modulePathArg.asSingle) + moduleOutputPath = try VirtualPath(path: modulePathArg) } else if moduleOutputKind == .topLevel { // FIXME: Logic to infer from primary outputs, etc. let moduleFilename = moduleName.appendingFileTypeExtension(.swiftModule) @@ -2556,7 +2964,7 @@ extension Driver { moduleOutputPath = try .init(path: moduleFilename) } } else { - moduleOutputPath = VirtualPath.createUniqueTemporaryFile(RelativePath(moduleName.appendingFileTypeExtension(.swiftModule))) + moduleOutputPath = try VirtualPath.createUniqueTemporaryFile(RelativePath(validating: moduleName.appendingFileTypeExtension(.swiftModule))) } // Use working directory if specified @@ -2577,8 +2985,10 @@ extension Driver { with moduleName: String, onError diagnosticsEngine: DiagnosticsEngine) -> [String: String]? { var moduleAliases: [String: String]? = nil - let validate = { (_ arg: String, allowModuleName: Bool) -> Bool in - if !arg.sd_isSwiftIdentifier { + // validatingModuleName should be true when validating the alias target (an actual module + // name), or false when validating the alias name (which can be a raw identifier). + let validate = { (_ arg: String, validatingModuleName: Bool) -> Bool in + if (validatingModuleName && !arg.sd_isSwiftIdentifier) || !arg.sd_isValidAsRawIdentifier { diagnosticsEngine.emit(.error_bad_module_name(moduleName: arg, explicitModuleName: true)) return false } @@ -2586,7 +2996,7 @@ extension Driver { diagnosticsEngine.emit(.error_stdlib_module_name(moduleName: arg, explicitModuleName: true)) return false } - if !allowModuleName, arg == moduleName { + if !validatingModuleName, arg == moduleName { diagnosticsEngine.emit(.error_bad_module_alias(arg, moduleName: moduleName)) return false } @@ -2639,17 +3049,13 @@ extension Driver { if let arg = parsedOptions.getLastArgument(.sdk) { sdkPath = arg.asSingle + } else if let SDKROOT = env["SDKROOT"] { + sdkPath = SDKROOT } else if compilerMode == .immediate || compilerMode == .repl { // In immediate modes, query the toolchain for a default SDK. - // We avoid using `SDKROOT` because that may be set to a compilation platform (like iOS) - // that is different than the host. sdkPath = try? toolchain.defaultSDKPath(targetTriple)?.pathString } - if sdkPath == nil, let SDKROOT = env["SDKROOT"] { - sdkPath = SDKROOT - } - // An empty string explicitly clears the SDK. if sdkPath == "" { sdkPath = nil @@ -2660,21 +3066,21 @@ extension Driver { // Validate the SDK if we found one. if let sdkPath = sdkPath { - let path: AbsolutePath + let path: VirtualPath // FIXME: TSC should provide a better utility for this. if let absPath = try? AbsolutePath(validating: sdkPath) { - path = absPath - } else if let cwd = fileSystem.currentWorkingDirectory, let absPath = try? AbsolutePath(validating: sdkPath, relativeTo: cwd) { - path = absPath + path = .absolute(absPath) + } else if let relPath = try? RelativePath(validating: sdkPath) { + path = .relative(relPath) } else { diagnosticsEngine.emit(.warning_no_such_sdk(sdkPath)) return nil } - if !fileSystem.exists(path) { + if (try? fileSystem.exists(path)) != true { diagnosticsEngine.emit(.warning_no_such_sdk(sdkPath)) - } else if !(targetTriple?.isWindows ?? (defaultToolchainType == WindowsToolchain.self)) { + } else if (targetTriple?.isDarwin ?? (defaultToolchainType == DarwinToolchain.self)) { if isSDKTooOld(sdkPath: path, fileSystem: fileSystem, diagnosticsEngine: diagnosticsEngine) { diagnosticsEngine.emit(.error_sdk_too_old(sdkPath)) @@ -2682,7 +3088,7 @@ extension Driver { } } - return .absolute(path) + return path } return nil @@ -2691,15 +3097,15 @@ extension Driver { // SDK checking: attempt to diagnose if the SDK we are pointed at is too old. extension Driver { - static func isSDKTooOld(sdkPath: AbsolutePath, fileSystem: FileSystem, + static func isSDKTooOld(sdkPath: VirtualPath, fileSystem: FileSystem, diagnosticsEngine: DiagnosticsEngine) -> Bool { - let sdkInfoReadAttempt = DarwinToolchain.readSDKInfo(fileSystem, VirtualPath.absolute(sdkPath).intern()) + let sdkInfoReadAttempt = DarwinToolchain.readSDKInfo(fileSystem, sdkPath.intern()) guard let sdkInfo = sdkInfoReadAttempt else { - diagnosticsEngine.emit(.warning_no_sdksettings_json(sdkPath.pathString)) + diagnosticsEngine.emit(.warning_no_sdksettings_json(sdkPath.name)) return false } guard let sdkVersion = try? Version(string: sdkInfo.versionString, lenient: true) else { - diagnosticsEngine.emit(.warning_fail_parse_sdk_ver(sdkInfo.versionString, sdkPath.pathString)) + diagnosticsEngine.emit(.warning_fail_parse_sdk_ver(sdkInfo.versionString, sdkPath.name)) return false } if sdkInfo.canonicalName.hasPrefix("macos") { @@ -2718,48 +3124,64 @@ extension Driver { // Imported Objective-C header. extension Driver { /// Compute the path of the imported Objective-C header. - static func computeImportedObjCHeader( + func computeImportedObjCHeader( _ parsedOptions: inout ParsedOptions, compilerMode: CompilerMode, - diagnosticEngine: DiagnosticsEngine - ) throws -> VirtualPath.Handle? { - guard let objcHeaderPathArg = parsedOptions.getLastArgument(.importObjcHeader) else { - return nil + chainedBridgingHeader: ChainedBridgingHeaderFile?) throws -> VirtualPath.Handle? { + // handle chained bridging header. + if let chainedHeader = chainedBridgingHeader, !chainedHeader.path.isEmpty { + let path = try VirtualPath(path: chainedHeader.path) + let dirExists = try fileSystem.exists(path.parentDirectory) + if !dirExists, let dirToCreate = path.parentDirectory.absolutePath { + try fileSystem.createDirectory(dirToCreate, recursive: true) + } + try fileSystem.writeFileContents(path, + bytes: ByteString(encodingAsUTF8: chainedHeader.content), + atomically: true) + return path.intern() } + return originalObjCHeaderFile + } - // Check for conflicting options. - if parsedOptions.hasArgument(.importUnderlyingModule) { - diagnosticEngine.emit(.error_framework_bridging_header) + /// Compute the path to the bridging precompiled header directory path. + func computePrecompiledBridgingHeaderDir( + _ parsedOptions: inout ParsedOptions, + compilerMode: CompilerMode) throws -> VirtualPath? { + if let outputPath = try? outputFileMap?.existingOutputForSingleInput(outputType: .pch) { + return VirtualPath.lookup(outputPath).parentDirectory } - - if parsedOptions.hasArgument(.emitModuleInterface, .emitModuleInterfacePath) { - diagnosticEngine.emit(.error_bridging_header_module_interface) + if let outputDir = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle { + return try VirtualPath(path: outputDir) } - - return try VirtualPath.intern(path: objcHeaderPathArg.asSingle) + return nil } /// Compute the path of the generated bridging PCH for the Objective-C header. - static func computeBridgingPrecompiledHeader(_ parsedOptions: inout ParsedOptions, - compilerMode: CompilerMode, - importedObjCHeader: VirtualPath.Handle?, - outputFileMap: OutputFileMap?) throws -> VirtualPath.Handle? { - guard compilerMode.supportsBridgingPCH, - let input = importedObjCHeader, - parsedOptions.hasFlag(positive: .enableBridgingPch, negative: .disableBridgingPch, default: true) else { + func computeBridgingPrecompiledHeader(_ parsedOptions: inout ParsedOptions, + compilerMode: CompilerMode, + importedObjCHeader: VirtualPath.Handle?, + outputFileMap: OutputFileMap?, + outputDirectory: VirtualPath?, + contextHash: String?) -> VirtualPath.Handle? { + guard compilerMode.supportsBridgingPCH, producePCHJob, let input = importedObjCHeader else { return nil } - if let outputPath = try outputFileMap?.existingOutput(inputFile: input, outputType: .pch) { + if let outputPath = try? outputFileMap?.existingOutputForSingleInput(outputType: .pch) { return outputPath } - let inputFile = VirtualPath.lookup(input) - let pchFileName = inputFile.basenameWithoutExt.appendingFileTypeExtension(.pch) - if let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle { - return try VirtualPath(path: outputDirectory).appending(component: pchFileName).intern() + let pchFile : String + let baseName = VirtualPath.lookup(input).basenameWithoutExt + if let hash = contextHash { + pchFile = baseName + "-" + hash + ".pch" } else { - return VirtualPath.createUniqueTemporaryFile(RelativePath(pchFileName)).intern() + pchFile = baseName.appendingFileTypeExtension(.pch) + } + if let outputDirectory = outputDirectory { + return outputDirectory.appending(component: pchFile).intern() + } else { + return try? VirtualPath.temporary(RelativePath(validating: pchFile)).intern() } } } @@ -2784,10 +3206,19 @@ extension Diagnostic.Message { extension Driver { static func validateWarningControlArgs(_ parsedOptions: inout ParsedOptions, diagnosticEngine: DiagnosticsEngine) { - if parsedOptions.hasArgument(.suppressWarnings) && - parsedOptions.hasFlag(positive: .warningsAsErrors, negative: .noWarningsAsErrors, default: false) { - diagnosticEngine.emit(.error(Error.conflictingOptions(.warningsAsErrors, .suppressWarnings)), - location: nil) + if parsedOptions.hasArgument(.suppressWarnings) { + if parsedOptions.hasFlag(positive: .warningsAsErrors, negative: .noWarningsAsErrors, default: false) { + diagnosticEngine.emit(.error(Error.conflictingOptions(.warningsAsErrors, .suppressWarnings)), + location: nil) + } + if parsedOptions.hasArgument(.Wwarning) { + diagnosticEngine.emit(.error(Error.conflictingOptions(.Wwarning, .suppressWarnings)), + location: nil) + } + if parsedOptions.hasArgument(.Werror) { + diagnosticEngine.emit(.error(Error.conflictingOptions(.Werror, .suppressWarnings)), + location: nil) + } } } @@ -2865,15 +3296,27 @@ extension Driver { fileSystem: FileSystem, workingDirectory: AbsolutePath?, diagnosticEngine: DiagnosticsEngine) { - if parsedOptions.hasArgument(.profileGenerate) && - parsedOptions.hasArgument(.profileUse) { - diagnosticEngine.emit(.error(Error.conflictingOptions(.profileGenerate, .profileUse)), - location: nil) + let conflictingProfArgs: [Option] = [.profileGenerate, + .profileUse, + .profileSampleUse] + + // Find out which of the mutually exclusive profiling arguments were provided. + let provided = conflictingProfArgs.filter { parsedOptions.hasArgument($0) } + + // If there's at least two of them, there's a conflict. + if provided.count >= 2 { + for i in 1.. Toolchain.Type { switch os { - case .darwin, .macosx, .ios, .tvos, .watchos: + case .darwin, .macosx, .ios, .tvos, .watchos, .visionos: return DarwinToolchain.self case .linux: return GenericUnixToolchain.self @@ -3053,23 +3523,53 @@ extension Driver { static func computeHostTriple( _ parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine, + libSwiftScan: SwiftScan?, toolchain: Toolchain, executor: DriverExecutor, fileSystem: FileSystem, - workingDirectory: AbsolutePath?, - swiftCompilerPrefixArgs: [String]) throws -> Triple { + workingDirectory: AbsolutePath?) throws -> Triple { let frontendOverride = try FrontendOverride(&parsedOptions, diagnosticsEngine) frontendOverride.setUpForTargetInfo(toolchain) defer { frontendOverride.setUpForCompilation(toolchain) } return try Self.computeTargetInfo(target: nil, targetVariant: nil, swiftCompilerPrefixArgs: frontendOverride.prefixArgsForTargetInfo, + libSwiftScan: libSwiftScan, toolchain: toolchain, fileSystem: fileSystem, workingDirectory: workingDirectory, diagnosticsEngine: diagnosticsEngine, executor: executor).target.triple } + static func initializeSwiftScanInstance( + _ parsedOptions: inout ParsedOptions, + diagnosticsEngine: DiagnosticsEngine, + toolchain: Toolchain, + interModuleDependencyOracle: InterModuleDependencyOracle, + fileSystem: FileSystem, + compilerIntegratedTooling: Bool) throws -> SwiftScan? { + guard !parsedOptions.hasArgument(.driverScanDependenciesNonLib) else { + return nil + } + + let swiftScanLibPath: AbsolutePath? = compilerIntegratedTooling ? nil : try toolchain.lookupSwiftScanLib() + do { + guard compilerIntegratedTooling || + (swiftScanLibPath != nil && fileSystem.exists(swiftScanLibPath!)) else { + diagnosticsEngine.emit(.warn_scan_dylib_not_found()) + return nil + } + + // Ensure the oracle initializes or verifies the existing scanner instance + try interModuleDependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: swiftScanLibPath) + // The driver needs a reference to this for non-scanning tasks + return interModuleDependencyOracle.getScannerInstance() + } catch { + diagnosticsEngine.emit(.warn_scan_dylib_load_failed(swiftScanLibPath?.description ?? "built-in")) + } + return nil + } + static func computeToolchain( _ parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine, @@ -3080,23 +3580,11 @@ extension Driver { useStaticResourceDir: Bool, workingDirectory: AbsolutePath?, compilerExecutableDir: AbsolutePath? - ) throws -> (Toolchain, FrontendTargetInfo, [String]) { + ) throws -> (Toolchain, [String]) { let explicitTarget = (parsedOptions.getLastArgument(.target)?.asSingle) .map { Triple($0, normalizing: true) } - let explicitTargetVariant = (parsedOptions.getLastArgument(.targetVariant)?.asSingle) - .map { - Triple($0, normalizing: true) - } - - // Determine the resource directory. - let resourceDirPath: VirtualPath? - if let resourceDirArg = parsedOptions.getLastArgument(.resourceDir) { - resourceDirPath = try VirtualPath(path: resourceDirArg.asSingle) - } else { - resourceDirPath = nil - } let toolchainType = try explicitTarget?.toolchainType(diagnosticsEngine) ?? defaultToolchainType @@ -3110,30 +3598,61 @@ extension Driver { compilerExecutableDir: compilerExecutableDir, toolDirectory: toolDir) + let frontendOverride = try FrontendOverride(&parsedOptions, diagnosticsEngine) + return (toolchain, frontendOverride.prefixArgs) + } + + static func computeTargetInfo(_ parsedOptions: inout ParsedOptions, + diagnosticsEngine: DiagnosticsEngine, + compilerMode: CompilerMode, + env: [String: String], + executor: DriverExecutor, + libSwiftScan: SwiftScan?, + toolchain: Toolchain, + fileSystem: FileSystem, + useStaticResourceDir: Bool, + workingDirectory: AbsolutePath?, + compilerExecutableDir: AbsolutePath?) throws -> FrontendTargetInfo { + let explicitTarget = (parsedOptions.getLastArgument(.target)?.asSingle) + .map { + Triple($0, normalizing: true) + } + let explicitTargetVariant = (parsedOptions.getLastArgument(.targetVariant)?.asSingle) + .map { + Triple($0, normalizing: true) + } + let frontendOverride = try FrontendOverride(&parsedOptions, diagnosticsEngine) frontendOverride.setUpForTargetInfo(toolchain) defer { frontendOverride.setUpForCompilation(toolchain) } + // Find the SDK, if any. let sdkPath: VirtualPath? = Self.computeSDKPath( &parsedOptions, compilerMode: compilerMode, toolchain: toolchain, targetTriple: explicitTarget, fileSystem: fileSystem, diagnosticsEngine: diagnosticsEngine, env: env) - // Query the frontend for target information. do { + // Determine the resource directory. + let resourceDirPath: VirtualPath? + if let resourceDirArg = parsedOptions.getLastArgument(.resourceDir) { + resourceDirPath = try VirtualPath(path: resourceDirArg.asSingle) + } else { + resourceDirPath = nil + } var info: FrontendTargetInfo = try Self.computeTargetInfo(target: explicitTarget, targetVariant: explicitTargetVariant, sdkPath: sdkPath, resourceDirPath: resourceDirPath, runtimeCompatibilityVersion: - parsedOptions.getLastArgument(.runtimeCompatibilityVersion)?.asSingle, + parsedOptions.getLastArgument(.runtimeCompatibilityVersion)?.asSingle, useStaticResourceDir: useStaticResourceDir, swiftCompilerPrefixArgs: frontendOverride.prefixArgsForTargetInfo, + libSwiftScan: libSwiftScan, toolchain: toolchain, fileSystem: fileSystem, workingDirectory: workingDirectory, diagnosticsEngine: diagnosticsEngine, executor: executor) - // Parse the runtime compatibility version. If present, it will override // what is reported by the frontend. if let versionString = @@ -3155,23 +3674,23 @@ extension Driver { diagnosticsEngine.emit(.warning_inferring_simulator_target(originalTriple: explicitTarget, inferredTriple: info.target.triple)) } - return (toolchain, info, frontendOverride.prefixArgs) + return info } catch let JobExecutionError.decodingError(decodingError, dataToDecode, processResult) { let stringToDecode = String(data: dataToDecode, encoding: .utf8) let errorDesc: String switch decodingError { - case let .typeMismatch(type, context): - errorDesc = "type mismatch: \(type), path: \(context.codingPath)" - case let .valueNotFound(type, context): - errorDesc = "value missing: \(type), path: \(context.codingPath)" - case let .keyNotFound(key, context): - errorDesc = "key missing: \(key), path: \(context.codingPath)" - case let .dataCorrupted(context): - errorDesc = "data corrupted at path: \(context.codingPath)" - @unknown default: - errorDesc = "unknown decoding error" + case let .typeMismatch(type, context): + errorDesc = "type mismatch: \(type), path: \(context.codingPath)" + case let .valueNotFound(type, context): + errorDesc = "value missing: \(type), path: \(context.codingPath)" + case let .keyNotFound(key, context): + errorDesc = "key missing: \(key), path: \(context.codingPath)" + case let .dataCorrupted(context): + errorDesc = "data corrupted at path: \(context.codingPath)" + @unknown default: + errorDesc = "unknown decoding error" } throw Error.unableToDecodeFrontendTargetInfo( stringToDecode, @@ -3339,6 +3858,7 @@ extension Driver { static func computeModuleDocOutputPath( _ parsedOptions: inout ParsedOptions, moduleOutputPath: VirtualPath.Handle?, + outputOption: Option, compilerOutputType: FileType?, compilerMode: CompilerMode, outputFileMap: OutputFileMap?, @@ -3348,7 +3868,7 @@ extension Driver { moduleOutputPath: moduleOutputPath, type: .swiftDocumentation, isOutput: .emitModuleDoc, - outputPath: .emitModuleDocPath, + outputPath: outputOption, compilerOutputType: compilerOutputType, compilerMode: compilerMode, outputFileMap: outputFileMap, @@ -3359,6 +3879,7 @@ extension Driver { static func computeModuleSourceInfoOutputPath( _ parsedOptions: inout ParsedOptions, moduleOutputPath: VirtualPath.Handle?, + outputOption: Option, compilerOutputType: FileType?, compilerMode: CompilerMode, outputFileMap: OutputFileMap?, @@ -3370,7 +3891,7 @@ extension Driver { moduleOutputPath: moduleOutputPath, type: .swiftSourceInfoFile, isOutput: .emitModuleSourceInfo, - outputPath: .emitModuleSourceInfoPath, + outputPath: outputOption, compilerOutputType: compilerOutputType, compilerMode: compilerMode, outputFileMap: outputFileMap, @@ -3465,21 +3986,23 @@ extension Driver { // CAS and Caching. extension Driver { - mutating func getCASPluginPath() throws -> AbsolutePath? { - if let pluginOpt = parsedOptions.getLastArgument(.casPluginOption)?.asSingle { - return try AbsolutePath(validating: pluginOpt.description) + static func getCASPluginPath(parsedOptions: inout ParsedOptions, + toolchain: Toolchain) throws -> AbsolutePath? { + if let pluginPath = parsedOptions.getLastArgument(.casPluginPath)?.asSingle { + return try AbsolutePath(validating: pluginPath.description) } return try toolchain.lookupToolchainCASPluginLib() } - mutating func getOnDiskCASPath() throws -> AbsolutePath? { + static func getOnDiskCASPath(parsedOptions: inout ParsedOptions, + toolchain: Toolchain) throws -> AbsolutePath? { if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { return try AbsolutePath(validating: casPathOpt.description) } return nil; } - mutating func getCASPluginOptions() throws -> [(String, String)] { + static func getCASPluginOptions(parsedOptions: inout ParsedOptions) throws -> [(String, String)] { var options : [(String, String)] = [] for opt in parsedOptions.arguments(for: .casPluginOption) { let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1) @@ -3490,4 +4013,18 @@ extension Driver { } return options } + + static func computeScanningPrefixMapper(_ parsedOptions: inout ParsedOptions) throws -> [AbsolutePath: AbsolutePath] { + var mapping: [AbsolutePath: AbsolutePath] = [:] + for opt in parsedOptions.arguments(for: .scannerPrefixMap) { + let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1) + if pluginArg.count != 2 { + throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle) + } + let key = try AbsolutePath(validating: String(pluginArg[0])) + let value = try AbsolutePath(validating: String(pluginArg[1])) + mapping[key] = value + } + return mapping + } } diff --git a/Sources/SwiftDriver/Driver/OutputFileMap.swift b/Sources/SwiftDriver/Driver/OutputFileMap.swift index fd653cba7..08e580df9 100644 --- a/Sources/SwiftDriver/Driver/OutputFileMap.swift +++ b/Sources/SwiftDriver/Driver/OutputFileMap.swift @@ -47,7 +47,7 @@ public struct OutputFileMap: Hashable, Codable { } // Form the virtual path. - return VirtualPath.createUniqueTemporaryFile(RelativePath(inputFile.basenameWithoutExt.appendingFileTypeExtension(outputType))).intern() + return try VirtualPath.createUniqueTemporaryFile(RelativePath(validating: inputFile.basenameWithoutExt.appendingFileTypeExtension(outputType))).intern() } public func existingOutput(inputFile: VirtualPath.Handle, outputType: FileType) throws -> VirtualPath.Handle? { @@ -140,16 +140,15 @@ public struct OutputFileMap: Hashable, Codable { /// Store the output file map at the given path. public func store( fileSystem: FileSystem, - file: AbsolutePath, - diagnosticEngine: DiagnosticsEngine + file: AbsolutePath ) throws { let encoder = JSONEncoder() #if os(Linux) || os(Android) - encoder.outputFormatting = [.prettyPrinted] + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] #else if #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) { - encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes] + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] } #endif diff --git a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift index f220ea5f2..4b409aca6 100644 --- a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift +++ b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift @@ -19,11 +19,12 @@ import WinSDK import Glibc #elseif canImport(Musl) import Musl +#elseif canImport(Bionic) +import Bionic #else #error("Missing libc or equivalent") #endif -import TSCBasic // <<< import class TSCBasic.DiagnosticsEngine import struct TSCBasic.Diagnostic import struct TSCBasic.ProcessResult @@ -79,7 +80,7 @@ import var TSCBasic.stdoutStream case .regular, .silent: break case .verbose: - stdoutStream <<< arguments.map { $0.spm_shellEscaped() }.joined(separator: " ") <<< "\n" + stdoutStream.send("\(arguments.map { $0.spm_shellEscaped() }.joined(separator: " "))\n") stdoutStream.flush() case .parsableOutput: let messages = constructJobBeganMessages(job: job, arguments: arguments, pid: pid) @@ -114,7 +115,7 @@ import var TSCBasic.stdoutStream let output = (try? result.utf8Output() + result.utf8stderrOutput()) ?? "" if !output.isEmpty { Driver.stdErrQueue.sync { - stderrStream <<< output + stderrStream.send(output) stderrStream.flush() } } @@ -137,7 +138,11 @@ import var TSCBasic.stdoutStream } #else case .signalled(let signal): +#if canImport(Bionic) + let errorMessage = String(cString: strsignal(signal)) +#else let errorMessage = strsignal(signal).map { String(cString: $0) } ?? "" +#endif messages = constructJobSignalledMessages(job: job, error: errorMessage, output: output, signal: signal, pid: pid).map { ParsableMessage(name: job.kind.rawValue, kind: .signalled($0)) @@ -169,8 +174,13 @@ import var TSCBasic.stdoutStream // FIXME: Do we need to do error handling here? Can this even fail? guard let json = try? message.toJSON() else { return } Driver.stdErrQueue.sync { - stderrStream <<< json.count <<< "\n" - stderrStream <<< String(data: json, encoding: .utf8)! <<< "\n" + stderrStream.send( + """ + \(json.count) + \(String(data: json, encoding: .utf8)!) + + """ + ) stderrStream.flush() } } diff --git a/Sources/SwiftDriver/Execution/ArgsResolver.swift b/Sources/SwiftDriver/Execution/ArgsResolver.swift index b74167dd0..5c84b3c2f 100644 --- a/Sources/SwiftDriver/Execution/ArgsResolver.swift +++ b/Sources/SwiftDriver/Execution/ArgsResolver.swift @@ -12,12 +12,10 @@ import class Foundation.NSLock -import TSCBasic // <<< import func TSCBasic.withTemporaryDirectory import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath - -@_implementationOnly import Yams +import struct TSCBasic.SHA256 /// How the resolver is to handle usage of response files public enum ResponseFileHandling { @@ -50,7 +48,7 @@ public final class ArgsResolver { // FIXME: withTemporaryDirectory uses FileManager.default, need to create a FileSystem.temporaryDirectory api. let tmpDir: AbsolutePath = try withTemporaryDirectory(removeTreeOnDeinit: false) { path in // FIXME: TSC removes empty directories even when removeTreeOnDeinit is false. This seems like a bug. - try fileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0 <<< "" } + try fileSystem.writeFileContents(path.appending(component: ".keep-directory")) { $0.send("") } return path } self.temporaryDirectory = .absolute(tmpDir) @@ -66,12 +64,16 @@ public final class ArgsResolver { public func resolveArgumentList(for job: Job, useResponseFiles: ResponseFileHandling = .heuristic) throws -> ([String], usingResponseFile: Bool) { let tool = try resolve(.path(job.tool)) - var arguments = [tool] + (try job.commandLine.map { try resolve($0) }) + var arguments = [tool] + (try resolveArgumentList(for: job.commandLine)) let usingResponseFile = try createResponseFileIfNeeded(for: job, resolvedArguments: &arguments, useResponseFiles: useResponseFiles) return (arguments, usingResponseFile) } + public func resolveArgumentList(for commandLine: [Job.ArgTemplate]) throws -> [String] { + return try commandLine.map { try resolve($0) } + } + @available(*, deprecated, message: "use resolveArgumentList(for:,useResponseFiles:,quotePaths:)") public func resolveArgumentList(for job: Job, forceResponseFiles: Bool, quotePaths: Bool = false) throws -> [String] { @@ -152,7 +154,7 @@ public final class ArgsResolver { if let absPath = path.absolutePath { try fileSystem.writeFileContents(absPath) { out in for path in contents { - try! out <<< unsafeResolve(path: path) <<< "\n" + try! out.send("\(unsafeResolve(path: path))\n") } } } @@ -162,39 +164,11 @@ public final class ArgsResolver { throws { // FIXME: Need a way to support this for distributed build systems... if let absPath = path.absolutePath { - // This uses Yams to escape and quote strings, but not to output the whole yaml file because - // it sometimes outputs mappings in explicit block format (https://yaml.org/spec/1.2/spec.html#id2798057) - // and the frontend (llvm) only seems to support implicit block format. - try fileSystem.writeFileContents(absPath) { out in - for (input, map) in outputFileMap.entries { - out <<< quoteAndEscape(path: VirtualPath.lookup(input)) <<< ":" - if map.isEmpty { - out <<< " {}\n" - } else { - out <<< "\n" - for (type, output) in map { - out <<< " " <<< type.name <<< ": " <<< quoteAndEscape(path: VirtualPath.lookup(output)) <<< "\n" - } - } - } - } + try outputFileMap.store(fileSystem: fileSystem, file: absPath) } } - private func quoteAndEscape(path: VirtualPath) -> String { - let inputNode = Node.scalar(Node.Scalar(try! unsafeResolve(path: path), - Tag(.str), .doubleQuoted)) - // Width parameter of -1 sets preferred line-width to unlimited so that no extraneous - // line-breaks will be inserted during serialization. - let string = try! Yams.serialize(node: inputNode, width: -1) - // Remove the newline from the end - return string.trimmingCharacters(in: .whitespacesAndNewlines) - } - private func createResponseFileIfNeeded(for job: Job, resolvedArguments: inout [String], useResponseFiles: ResponseFileHandling) throws -> Bool { - func quote(_ string: String) -> String { - return "\"\(String(string.flatMap { ["\\", "\""].contains($0) ? "\\\($0)" : "\($0)" }))\"" - } guard useResponseFiles != .disabled else { return false } @@ -205,17 +179,15 @@ public final class ArgsResolver { assert(!forceResponseFiles || job.supportsResponseFiles, "Platform does not support response files for job: \(job)") // Match the integrated driver's behavior, which uses response file names of the form "arguments-[0-9a-zA-Z].resp". - let responseFilePath = temporaryDirectory.appending(component: "arguments-\(abs(job.hashValue)).resp") + let hash = SHA256().hash(resolvedArguments.joined(separator: " ")).hexadecimalRepresentation + let responseFilePath = temporaryDirectory.appending(component: "arguments-\(hash).resp") // FIXME: Need a way to support this for distributed build systems... if let absPath = responseFilePath.absolutePath { - // Adopt the same technique as clang - - // Wrap all arguments in double quotes to ensure that both Unix and - // Windows tools understand the response file. try fileSystem.writeFileContents(absPath) { - $0 <<< resolvedArguments[1...].map { quote($0) }.joined(separator: "\n") + $0.send(resolvedArguments[2...].map { $0.spm_shellEscaped() }.joined(separator: "\n")) } - resolvedArguments = [resolvedArguments[0], "@\(absPath.pathString)"] + resolvedArguments = [resolvedArguments[0], resolvedArguments[1], "@\(absPath.pathString)"] } return true diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift index 3968b1941..804119003 100644 --- a/Sources/SwiftDriver/Execution/DriverExecutor.swift +++ b/Sources/SwiftDriver/Execution/DriverExecutor.swift @@ -29,6 +29,7 @@ public protocol DriverExecutor { /// Execute multiple jobs, tracking job status using the provided execution delegate. /// Pass in the `IncrementalCompilationState` to allow for incremental compilation. + /// Pass in the `InterModuleDependencyGraph` to allow for module dependency tracking. func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, numParallelJobs: Int, @@ -58,24 +59,33 @@ public struct DriverExecutorWorkload { case all([Job]) case incremental(IncrementalCompilationState) } + public let kind: Kind + @available(*, deprecated, message: "use of 'interModuleDependencyGraph' on 'DriverExecutorWorkload' is deprecated") + public let interModuleDependencyGraph: InterModuleDependencyGraph? = nil + public init(_ allJobs: [Job], - _ incrementalCompilationState: IncrementalCompilationState?, - continueBuildingAfterErrors: Bool - ) { - self.continueBuildingAfterErrors = continueBuildingAfterErrors - self.kind = incrementalCompilationState - .map {.incremental($0)} - ?? .all(allJobs) + _ incrementalCompilationState: IncrementalCompilationState?, + continueBuildingAfterErrors: Bool) { + self.continueBuildingAfterErrors = continueBuildingAfterErrors + self.kind = incrementalCompilationState + .map {.incremental($0)} + ?? .all(allJobs) } static public func all(_ jobs: [Job]) -> Self { .init(jobs, nil, continueBuildingAfterErrors: false) } + + @available(*, deprecated, message: "use all(_ jobs: [Job]) instead") + static public func all(_ jobs: [Job], + _ interModuleDependencyGraph: InterModuleDependencyGraph?) -> Self { + .init(jobs, nil, continueBuildingAfterErrors: false) + } } -enum JobExecutionError: Error { +@_spi(Testing) public enum JobExecutionError: Error { case jobFailedWithNonzeroExitCode(Int, String) case failedToReadJobOutput // A way to pass more information to the catch point diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift deleted file mode 100644 index 4d83d0c85..000000000 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift +++ /dev/null @@ -1,199 +0,0 @@ -//===-------------- ClangVersionedDependencyResolution.swift --------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import func TSCBasic.determineTempDirectory - -/// A map from a module identifier to a set of module dependency graphs -/// Used to compute distinct graphs corresponding to different target versions for a given clang module -public typealias ModuleVersionedGraphMap = [ModuleDependencyId: [InterModuleDependencyGraph]] - -internal extension Driver { - // Dependency scanning results may vary depending on the target version specified on the - // dependency scanning action. If the scanning action is performed at a fixed target, and the - // scanned module is later compiled with a higher version target, miscomputation may occur - // due to dependencies present only at the higher version number and thus not detected by - // the dependency scanner. We must ensure to re-scan Clang modules at all targets at which - // they will be compiled and record a super-set of the module's dependencies at all targets. - /// For each clang module, compute its dependencies at all targets at which it will be compiled. - mutating func resolveVersionedClangDependencies(dependencyGraph: inout InterModuleDependencyGraph) - throws { - // Traverse the dependency graph, collecting extraPCMArgs along each path - // to all Clang modules, and compute a set of distinct PCMArgs across all paths to a - // given Clang module in the graph. - let modulePCMArgsSetMap = try dependencyGraph.computePCMArgSetsForClangModules() - - // Set up the batch scan input - let temporaryDirectory = try determineTempDirectory() - let batchScanInputList = - try modulePCMArgsSetMap.compactMap { (moduleId, pcmArgsSet) throws -> [BatchScanModuleInfo] in - var moduleInfos: [BatchScanModuleInfo] = [] - for pcmArgs in pcmArgsSet { - var hasher = Hasher() - pcmArgs.forEach { hasher.combine($0) } - // Generate a filepath for the output dependency graph - let moduleDependencyGraphPath = - temporaryDirectory.appending(component: moduleId.moduleName + - String(hasher.finalize()) + - "-dependencies.json") - let moduleBatchInfo = - BatchScanModuleInfo.clang( - BatchScanClangModuleInfo(moduleName: moduleId.moduleName, - pcmArgs: pcmArgs.joined(separator: " "), - outputPath: moduleDependencyGraphPath.description)) - moduleInfos.append(moduleBatchInfo) - } - return moduleInfos - }.reduce([], +) - - guard !batchScanInputList.isEmpty else { - // If no new re-scans are needed, we are done here. - return - } - - // Batch scan all clang modules for each discovered new, unique set of PCMArgs, per module - let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = - try performBatchDependencyScan(moduleInfos: batchScanInputList) - - // Update the dependency graph to reflect the newly-discovered dependencies - try dependencyGraph.resolveVersionedClangModules(using: moduleVersionedGraphMap) - try dependencyGraph.updateCapturedPCMArgClangDependencies(using: modulePCMArgsSetMap) - } -} - -private extension InterModuleDependencyGraph { - /// For each module scanned at multiple target versions, combine their dependencies across version-specific graphs. - mutating func resolveVersionedClangModules(using versionedGraphMap: ModuleVersionedGraphMap) - throws { - // Process each re-scanned module and its collection of graphs - for (moduleId, graphList) in versionedGraphMap { - for versionedGraph in graphList { - // We must update dependencies for each module in the versioned graph, not just - // the top-level re-scanned module itself. - for rescannedModuleId in versionedGraph.modules.keys { - guard let versionedModuleInfo = versionedGraph.modules[rescannedModuleId] else { - throw Driver.Error.missingModuleDependency(moduleId.moduleName) - } - // If the main graph already contains this module, update its dependencies - if var currentModuleInfo = modules[rescannedModuleId] { - versionedModuleInfo.directDependencies?.forEach { dependencyId in - // If a not-seen-before dependency has been found, add it to the info - if !currentModuleInfo.directDependencies!.contains(dependencyId) { - currentModuleInfo.directDependencies!.append(dependencyId) - } - } - // Update the moduleInfo with the one whose dependencies consist of a super-set - // of dependencies across all of the versioned dependency graphs - modules[rescannedModuleId] = currentModuleInfo - } else { - // If the main graph does not yet contain this module, add it to the graph - modules[rescannedModuleId] = versionedModuleInfo - } - } - } - } - } - - /// DFS from the main module to all clang modules, accumulating distinct - /// PCMArgs along all paths to a given Clang module - func computePCMArgSetsForClangModules() throws -> [ModuleDependencyId : Set<[String]>] { - let mainModuleId: ModuleDependencyId = .swift(mainModuleName) - var pcmArgSetMap: [ModuleDependencyId : Set<[String]>] = [:] - - var visitedSwiftModules: Set = [] - - func visit(_ moduleId: ModuleDependencyId, - pathPCMArtSet: Set<[String]>, - pcmArgSetMap: inout [ModuleDependencyId : Set<[String]>]) - throws { - guard let moduleInfo = modules[moduleId] else { - throw Driver.Error.missingModuleDependency(moduleId.moduleName) - } - switch moduleId { - case .swift: - if visitedSwiftModules.contains(moduleId) { - return - } else { - visitedSwiftModules.insert(moduleId) - } - guard case .swift(let swiftModuleDetails) = moduleInfo.details else { - throw Driver.Error.malformedModuleDependency(moduleId.moduleName, - "no Swift `details` object") - } - // Add extraPCMArgs of the visited node to the current path set - // and proceed to visit all direct dependencies - let modulePCMArgs = swiftModuleDetails.extraPcmArgs - var newPathPCMArgSet = pathPCMArtSet - newPathPCMArgSet.insert(modulePCMArgs) - for dependencyId in moduleInfo.directDependencies! { - try visit(dependencyId, - pathPCMArtSet: newPathPCMArgSet, - pcmArgSetMap: &pcmArgSetMap) - } - case .clang: - guard case .clang(let clangModuleDetails) = moduleInfo.details else { - throw Driver.Error.malformedModuleDependency(moduleId.moduleName, - "no Clang `details` object") - } - // The details of this module contain information on which sets of PCMArgs are already - // captured in the described dependencies of this module. Only re-scan at PCMArgs not - // already captured. - let alreadyCapturedPCMArgs = - clangModuleDetails.capturedPCMArgs ?? Set<[String]>() - let newPCMArgSet = pathPCMArtSet.filter { !alreadyCapturedPCMArgs.contains($0) } - // Add current path's PCMArgs to the SetMap and stop traversal - if pcmArgSetMap[moduleId] != nil { - newPCMArgSet.forEach { pcmArgSetMap[moduleId]!.insert($0) } - } else { - pcmArgSetMap[moduleId] = newPCMArgSet - } - return - case .swiftPrebuiltExternal: - // We can rely on the fact that this pre-built module already has its - // versioned-PCM dependencies satisfied, so we do not need to add additional - // arguments. Proceed traversal to its dependencies. - for dependencyId in moduleInfo.directDependencies! { - try visit(dependencyId, - pathPCMArtSet: pathPCMArtSet, - pcmArgSetMap: &pcmArgSetMap) - } - case .swiftPlaceholder: - fatalError("Unresolved placeholder dependencies at planning stage: \(moduleId)") - } - } - - try visit(mainModuleId, - pathPCMArtSet: [], - pcmArgSetMap: &pcmArgSetMap) - return pcmArgSetMap - } - - /// Update the set of all PCMArgs against which a given clang module was re-scanned - mutating func updateCapturedPCMArgClangDependencies(using pcmArgSetMap: - [ModuleDependencyId : Set<[String]>] - ) throws { - for (moduleId, newPCMArgs) in pcmArgSetMap { - guard let moduleInfo = modules[moduleId] else { - throw Driver.Error.missingModuleDependency(moduleId.moduleName) - } - guard case .clang(var clangModuleDetails) = moduleInfo.details else { - throw Driver.Error.malformedModuleDependency(moduleId.moduleName, - "no Clang `details` object") - } - if clangModuleDetails.capturedPCMArgs == nil { - clangModuleDetails.capturedPCMArgs = Set<[String]>() - } - newPCMArgs.forEach { clangModuleDetails.capturedPCMArgs!.insert($0) } - modules[moduleId]!.details = .clang(clangModuleDetails) - } - } -} - diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index 3b4b1fcf7..34ccbaffc 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -27,6 +27,12 @@ public struct ExternalTargetModuleDetails { let isFramework: Bool } +/// A chained bridging header file. +public struct ChainedBridgingHeaderFile { + let path: String + let content: String +} + public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalTargetModuleDetails] /// In Explicit Module Build mode, this planner is responsible for generating and providing @@ -34,7 +40,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// that specify said explicit module dependencies. @_spi(Testing) public struct ExplicitDependencyBuildPlanner { /// The module dependency graph. - private let dependencyGraph: InterModuleDependencyGraph + @_spi(Testing) public let dependencyGraph: InterModuleDependencyGraph /// A set of direct and transitive dependencies for every module in the dependency graph private let reachabilityMap: [ModuleDependencyId : Set] @@ -45,8 +51,9 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// Whether we are using the integrated driver via libSwiftDriver shared lib private let integratedDriver: Bool private let mainModuleName: String? - private let enableCAS: Bool + private let cas: SwiftScanCAS? private let swiftScanOracle: InterModuleDependencyOracle + private let prefixMap: [(AbsolutePath, AbsolutePath)] /// Clang PCM names contain a hash of the command-line arguments that were used to build them. /// We avoid re-running the hash computation with the use of this cache @@ -55,12 +62,20 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// Does this compile support `.explicitInterfaceModuleBuild` private var supportsExplicitInterfaceBuild: Bool + /// Cached command-line additions for all main module compile jobs + private struct ResolvedModuleDependenciesCommandLineComponents { + let inputs: [TypedVirtualPath] + let commandLine: [Job.ArgTemplate] + } + private var resolvedMainModuleDependenciesArgs: ResolvedModuleDependenciesCommandLineComponents? = nil + public init(dependencyGraph: InterModuleDependencyGraph, toolchain: Toolchain, dependencyOracle: InterModuleDependencyOracle, integratedDriver: Bool = true, supportsExplicitInterfaceBuild: Bool = false, - enableCAS: Bool = false) throws { + cas: SwiftScanCAS? = nil, + prefixMap: [(AbsolutePath, AbsolutePath)] = []) throws { self.dependencyGraph = dependencyGraph self.toolchain = toolchain self.swiftScanOracle = dependencyOracle @@ -68,12 +83,13 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT self.mainModuleName = dependencyGraph.mainModuleName self.reachabilityMap = try dependencyGraph.computeTransitiveClosure() self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild - self.enableCAS = enableCAS + self.cas = cas + self.prefixMap = prefixMap } /// Supports resolving bridging header pch command from swiftScan. - public func supportsBridgingHeaderPCHCommand() throws -> Bool { - return try swiftScanOracle.supportsBridgingHeaderPCHCommand() + public var supportsBridgingHeaderPCHCommand: Bool { + return swiftScanOracle.supportsBridgingHeaderPCHCommand } /// Generate build jobs for all dependencies of the main module. @@ -136,9 +152,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT for moduleId in swiftDependencies { let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId) var inputs: [TypedVirtualPath] = [] - let outputs: [TypedVirtualPath] = [ - TypedVirtualPath(file: moduleInfo.modulePath.path, type: .swiftModule) - ] var commandLine: [Job.ArgTemplate] = [] // First, take the command line options provided in the dependency information let moduleDetails = try dependencyGraph.swiftModuleDetails(of: moduleId) @@ -155,9 +168,18 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT throw Driver.Error.malformedModuleDependency(moduleId.moduleName, "no `moduleInterfacePath` object") } - inputs.append(TypedVirtualPath(file: moduleInterfacePath.path, - type: .swiftInterface)) + let inputInterfacePath = TypedVirtualPath(file: moduleInterfacePath.path, type: .swiftInterface) + inputs.append(inputInterfacePath) + let outputModulePath = TypedVirtualPath(file: moduleInfo.modulePath.path, type: .swiftModule) + let outputs = [outputModulePath] + + let cacheKeys : [TypedVirtualPath : String] + if let key = moduleDetails.moduleCacheKey { + cacheKeys = [inputInterfacePath: key] + } else { + cacheKeys = [:] + } // Add precompiled module candidates, if present if let compiledCandidateList = moduleDetails.compiledModuleCandidates { for compiledCandidate in compiledCandidateList { @@ -166,6 +188,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT } } + // Add prefix mapping. The option is cache invariant so it can be added without affecting cache key. + for (key, value) in prefixMap { + commandLine.appendFlag("-cache-replay-prefix-map") + commandLine.appendFlag(value.pathString + "=" + key.pathString) + } + jobs.append(Job( moduleName: moduleId.moduleName, kind: .compileModuleFromInterface, @@ -173,14 +201,14 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT commandLine: commandLine, inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys )) } return jobs } - /// Generate a build job for each Clang module in the set of supplied `dependencies`. Once per each required - /// PCMArgSet as queried from the supplied `clangPCMSetMap` + /// Generate a build job for each Clang module in the set of supplied `dependencies`. private mutating func generateClangDependenciesBuildJobs(for dependencies: Set) throws -> [Job] { var jobs: [Job] = [] @@ -205,15 +233,25 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT try resolveExplicitModuleDependencies(moduleId: moduleId, inputs: &inputs, commandLine: &commandLine) - let moduleMapPath = moduleDetails.moduleMapPath.path - let modulePCMPath = moduleInfo.modulePath - outputs.append(TypedVirtualPath(file: modulePCMPath.path, type: .pcm)) + let moduleMapPath = TypedVirtualPath(file: moduleDetails.moduleMapPath.path, type: .clangModuleMap) + let modulePCMPath = TypedVirtualPath(file: moduleInfo.modulePath.path, type: .pcm) + outputs.append(modulePCMPath) // The only required input is the .modulemap for this module. // Command line options in the dependency scanner output will include the // required modulemap, so here we must only add it to the list of inputs. - inputs.append(TypedVirtualPath(file: moduleMapPath, - type: .clangModuleMap)) + let cacheKeys : [TypedVirtualPath : String] + if let key = moduleDetails.moduleCacheKey { + cacheKeys = [moduleMapPath: key] + } else { + cacheKeys = [:] + } + + // Add prefix mapping. The option is cache invariant so it can be added without affecting cache key. + for (key, value) in prefixMap { + commandLine.appendFlag("-cache-replay-prefix-map") + commandLine.appendFlag(value.pathString + "=" + key.pathString) + } jobs.append(Job( moduleName: moduleId.moduleName, @@ -222,7 +260,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT commandLine: commandLine, inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys )) } return jobs @@ -234,8 +273,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { // Prohibit the frontend from implicitly building textual modules into binary modules. - var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] - var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] + var swiftDependencyArtifacts: Set = [] + var clangDependencyArtifacts: Set = [] try addModuleDependencies(of: moduleId, clangDependencyArtifacts: &clangDependencyArtifacts, swiftDependencyArtifacts: &swiftDependencyArtifacts) @@ -245,29 +284,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT for dependencyModule in swiftDependencyArtifacts { inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path, type: .swiftModule)) - - let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? [] - if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty { - throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency") - } - - for headerDep in prebuiltHeaderDependencyPaths { - commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"]) - commandLine.appendPath(VirtualPath.lookup(headerDep.path)) - inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch)) - } } - - // Clang module dependencies are specified on the command line explicitly for moduleArtifactInfo in clangDependencyArtifacts { let clangModulePath = TypedVirtualPath(file: moduleArtifactInfo.clangModulePath.path, type: .pcm) - let clangModuleMapPath = - TypedVirtualPath(file: moduleArtifactInfo.clangModuleMapPath.path, - type: .clangModuleMap) inputs.append(clangModulePath) - inputs.append(clangModuleMapPath) } // Swift Main Module dependencies are passed encoded in a JSON file as described by @@ -277,16 +299,16 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT try serializeModuleDependencies(for: moduleId, swiftDependencyArtifacts: swiftDependencyArtifacts, clangDependencyArtifacts: clangDependencyArtifacts) - if enableCAS { + if let cas = self.cas { // When using a CAS, write JSON into CAS and pass the ID on command-line. - let casID = try swiftScanOracle.store(data: dependencyFileContent) + let casID = try cas.store(data: dependencyFileContent) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendFlag(casID) } else { // Write JSON to a file and add the JSON artifacts to command-line and inputs. let dependencyFile = - VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), - dependencyFileContent) + try VirtualPath.createUniqueTemporaryFileWithKnownContents(.init(validating: "\(moduleId.moduleName)-dependencies.json"), + dependencyFileContent) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendPath(dependencyFile) inputs.append(TypedVirtualPath(file: dependencyFile.intern(), @@ -296,8 +318,9 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT private mutating func addModuleDependency(of moduleId: ModuleDependencyId, dependencyId: ModuleDependencyId, - clangDependencyArtifacts: inout [ClangModuleArtifactInfo], - swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + clangDependencyArtifacts: inout Set, + swiftDependencyArtifacts: inout Set, + bridgingHeaderDeps: Set? = nil ) throws { switch dependencyId { case .swift: @@ -310,7 +333,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT isFramework = swiftModuleDetails.isFramework ?? false // Accumulate the required information about this dependency // TODO: add .swiftdoc and .swiftsourceinfo for this module. - swiftDependencyArtifacts.append( + swiftDependencyArtifacts.insert( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), isFramework: isFramework, @@ -320,11 +343,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT let dependencyClangModuleDetails = try dependencyGraph.clangModuleDetails(of: dependencyId) // Accumulate the required information about this dependency - clangDependencyArtifacts.append( + clangDependencyArtifacts.insert( ClangModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path), moduleMapPath: dependencyClangModuleDetails.moduleMapPath, - moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey)) + moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey, + isBridgingHeaderDependency: bridgingHeaderDeps?.contains(dependencyId) ?? true)) case .swiftPrebuiltExternal: let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId) let compiledModulePath = prebuiltModuleDetails.compiledModulePath @@ -333,7 +357,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT .init(file: compiledModulePath.path, type: .swiftModule) // Accumulate the required information about this dependency // TODO: add .swiftdoc and .swiftsourceinfo for this module. - swiftDependencyArtifacts.append( + swiftDependencyArtifacts.insert( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), headerDependencies: prebuiltModuleDetails.headerDependencyPaths, @@ -344,11 +368,37 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT } } + /// Collect the Set of all Clang module dependencies which are dependencies of either + /// the `moduleId` bridging header or dependencies of bridging headers + /// of any prebuilt binary Swift modules in the dependency graph. + private func collectHeaderModuleDeps(of moduleId: ModuleDependencyId) throws -> Set? { + var bridgingHeaderDeps: Set? = nil + guard let moduleDependencies = reachabilityMap[moduleId] else { + fatalError("Expected reachability information for the module: \(moduleId.moduleName).") + } + if let dependencySourceBridingHeaderDeps = + try dependencyGraph.moduleInfo(of: moduleId).bridgingHeaderModuleDependencies { + bridgingHeaderDeps = Set(dependencySourceBridingHeaderDeps) + } else { + bridgingHeaderDeps = Set() + } + // Collect all binary Swift module dependnecies' header input module dependencies + for dependencyId in moduleDependencies { + if case .swiftPrebuiltExternal(_) = dependencyId { + let prebuiltDependencyDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId) + for headerDependency in prebuiltDependencyDetails.headerDependencyModuleDependencies ?? [] { + bridgingHeaderDeps!.insert(headerDependency) + } + } + } + return bridgingHeaderDeps + } + /// Add a specific module dependency as an input and a corresponding command /// line flag. private mutating func addModuleDependencies(of moduleId: ModuleDependencyId, - clangDependencyArtifacts: inout [ClangModuleArtifactInfo], - swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + clangDependencyArtifacts: inout Set, + swiftDependencyArtifacts: inout Set ) throws { guard let moduleDependencies = reachabilityMap[moduleId] else { fatalError("Expected reachability information for the module: \(moduleId.moduleName).") @@ -356,45 +406,71 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT for dependencyId in moduleDependencies { try addModuleDependency(of: moduleId, dependencyId: dependencyId, clangDependencyArtifacts: &clangDependencyArtifacts, - swiftDependencyArtifacts: &swiftDependencyArtifacts) + swiftDependencyArtifacts: &swiftDependencyArtifacts, + bridgingHeaderDeps: try collectHeaderModuleDeps(of: moduleId)) } } - private func updateClangPCMArgSetMap(for moduleId: ModuleDependencyId, - clangPCMSetMap: inout [ModuleDependencyId : Set<[String]>]) - throws { - guard let moduleDependencies = reachabilityMap[moduleId] else { - fatalError("Expected reachability information for the module: \(moduleId.moduleName).") - } - let pcmArgs = try dependencyGraph.swiftModulePCMArgs(of: moduleId) - for dependencyId in moduleDependencies { - guard case .clang(_) = dependencyId else { + public func getLinkLibraryLoadCommandFlags(_ commandLine: inout [Job.ArgTemplate]) throws { + var allLinkLibraries: [LinkLibraryInfo] = [] + for (_, moduleInfo) in dependencyGraph.modules { + guard let moduleLinkLibraries = moduleInfo.linkLibraries else { continue } - if clangPCMSetMap[dependencyId] != nil { - clangPCMSetMap[dependencyId]!.insert(pcmArgs) - } else { - clangPCMSetMap[dependencyId] = [pcmArgs] + for linkLibrary in moduleLinkLibraries { + allLinkLibraries.append(linkLibrary) } } + toolchain.addAutoLinkFlags(for: allLinkLibraries, to: &commandLine) } /// Resolve all module dependencies of the main module and add them to the lists of /// inputs and command line flags. public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { + // If not previously computed, gather all dependency input files and command-line arguments + if resolvedMainModuleDependenciesArgs == nil { + var inputAdditions: [TypedVirtualPath] = [] + var commandLineAdditions: [Job.ArgTemplate] = [] + let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) + let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) + if let additionalArgs = mainModuleDetails.commandLine { + additionalArgs.forEach { commandLineAdditions.appendFlag($0) } + } + commandLineAdditions.appendFlags("-disable-implicit-swift-modules", + "-Xcc", "-fno-implicit-modules", + "-Xcc", "-fno-implicit-module-maps") + try resolveExplicitModuleDependencies(moduleId: mainModuleId, + inputs: &inputAdditions, + commandLine: &commandLineAdditions) + resolvedMainModuleDependenciesArgs = ResolvedModuleDependenciesCommandLineComponents( + inputs: inputAdditions, + commandLine: commandLineAdditions + ) + } + guard let mainModuleDependenciesArgs = resolvedMainModuleDependenciesArgs else { + fatalError("Failed to compute resolved explicit dependency arguments.") + } + inputs.append(contentsOf: mainModuleDependenciesArgs.inputs) + commandLine.append(contentsOf: mainModuleDependenciesArgs.commandLine) + } + + /// Get the context hash for the main module. + public func getMainModuleContextHash() throws -> String? { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) + let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) + return mainModuleDetails.contextHash + } + /// Get the chained bridging header info + public func getChainedBridgingHeaderFile() throws -> ChainedBridgingHeaderFile? { + let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) - if let additionalArgs = mainModuleDetails.commandLine { - additionalArgs.forEach { commandLine.appendFlag($0) } + guard let path = mainModuleDetails.chainedBridgingHeaderPath, + let content = mainModuleDetails.chainedBridgingHeaderContent else{ + return nil } - commandLine.appendFlags("-disable-implicit-swift-modules", - "-Xcc", "-fno-implicit-modules", - "-Xcc", "-fno-implicit-module-maps") - try resolveExplicitModuleDependencies(moduleId: mainModuleId, - inputs: &inputs, - commandLine: &commandLine) + return ChainedBridgingHeaderFile(path: path, content: content) } /// Resolve all module dependencies of the main module and add them to the lists of @@ -402,8 +478,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName) - var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = [] - var clangDependencyArtifacts: [ClangModuleArtifactInfo] = [] + var swiftDependencyArtifacts: Set = [] + var clangDependencyArtifacts: Set = [] let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId) var addedDependencies: Set = [] @@ -424,7 +500,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT clangDependencyArtifacts: &clangDependencyArtifacts, swiftDependencyArtifacts: &swiftDependencyArtifacts) let depInfo = try dependencyGraph.moduleInfo(of: bridgingHeaderDepID) - dependenciesWorklist.append(contentsOf: depInfo.directDependencies ?? []) + dependenciesWorklist.append(contentsOf: depInfo.allDependencies) } // Clang module dependencies are specified on the command line explicitly @@ -432,11 +508,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT let clangModulePath = TypedVirtualPath(file: moduleArtifactInfo.clangModulePath.path, type: .pcm) - let clangModuleMapPath = - TypedVirtualPath(file: moduleArtifactInfo.clangModuleMapPath.path, - type: .clangModuleMap) inputs.append(clangModulePath) - inputs.append(clangModuleMapPath) } // Return if depscanner provided build commands. @@ -445,7 +517,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT return } - assert(!enableCAS, "Caching build should always return command-line from scanner") + assert(cas == nil, "Caching build should always return command-line from scanner") // Prohibit the frontend from implicitly building textual modules into binary modules. commandLine.appendFlags("-disable-implicit-swift-modules", "-Xcc", "-fno-implicit-modules", @@ -457,8 +529,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT clangDependencyArtifacts: clangDependencyArtifacts) let dependencyFile = - VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(mainModuleId.moduleName)-dependencies.json"), - dependencyFileContent) + try VirtualPath.createUniqueTemporaryFileWithKnownContents(.init(validating: "\(mainModuleId.moduleName)-dependencies.json"), + dependencyFileContent) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendPath(dependencyFile) inputs.append(TypedVirtualPath(file: dependencyFile.intern(), @@ -467,8 +539,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT /// Serialize the output file artifacts for a given module in JSON format. private func serializeModuleDependencies(for moduleId: ModuleDependencyId, - swiftDependencyArtifacts: [SwiftModuleArtifactInfo], - clangDependencyArtifacts: [ClangModuleArtifactInfo] + swiftDependencyArtifacts: Set, + clangDependencyArtifacts: Set ) throws -> Data { // The module dependency map in CAS needs to be stable. // Sort the dependencies by name. @@ -480,26 +552,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT encoder.outputFormatting = [.prettyPrinted, .sortedKeys] return try encoder.encode(allDependencyArtifacts) } - - private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] { - var results: [String] = [] - results.append(contextHash) - results.append(contentsOf: pcmArgs) - if integratedDriver { - return results - } - - // We need this to enable explicit modules in the driver-as-executable mode. For instance, - // we have two Swift targets A and B, where A depends on X.pcm which in turn depends on Y.pcm, - // and B only depends on Y.pcm. In the driver-as-executable mode, the build system isn't aware - // of the shared dependency of Y.pcm so it will be generated multiple times. If all these Y.pcm - // share the same name, X.pcm may fail to be loaded because its dependency Y.pcm may have a - // changed mod time. - // We only differentiate these PCM names in the non-integrated mode due to the lacking of - // inter-module planning. - results.append(mainModuleName!) - return results - } } /// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking @@ -535,16 +587,22 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT } return clangModuleDetails } - - func swiftModulePCMArgs(of moduleId: ModuleDependencyId) throws -> [String] { - let moduleDetails = try swiftModuleDetails(of: moduleId) - return moduleDetails.extraPcmArgs - } } internal extension ExplicitDependencyBuildPlanner { - func explainDependency(_ dependencyModuleName: String) throws -> [[ModuleDependencyId]]? { - return try dependencyGraph.explainDependency(dependencyModuleName: dependencyModuleName) + func explainDependency(_ dependencyModuleName: String, allPaths: Bool) throws -> [[ModuleDependencyId]]? { + return try dependencyGraph.explainDependency(dependencyModuleName: dependencyModuleName, allPaths: allPaths) + } + + func findPath(from source: ModuleDependencyId, to destination: ModuleDependencyId) throws -> [ModuleDependencyId]? { + guard dependencyGraph.modules.contains(where: { $0.key == destination }) else { return nil } + var result: [ModuleDependencyId]? = nil + var visited: Set = [] + try dependencyGraph.findAPath(source: source, + pathSoFar: [source], + visited: &visited, + result: &result) { $0 == destination } + return result } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift index a4c742a07..9ea3b5e1b 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import func TSCBasic.topologicalSort +import protocol TSCBasic.FileSystem @_spi(Testing) public extension InterModuleDependencyGraph { /// For targets that are built alongside the driver's current module, the scanning action will report them as @@ -27,24 +28,26 @@ import func TSCBasic.topologicalSort if let currentInfo = modules[swiftModuleId], externalModuleId.moduleName != mainModuleName { let newExternalModuleDetails = - try SwiftPrebuiltExternalModuleDetails(compiledModulePath: - TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), - isFramework: externalModuleDetails.isFramework) + SwiftPrebuiltExternalModuleDetails(compiledModulePath: + TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), + isFramework: externalModuleDetails.isFramework) let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), sourceFiles: [], directDependencies: currentInfo.directDependencies, + linkLibraries: currentInfo.linkLibraries, details: .swiftPrebuiltExternal(newExternalModuleDetails)) Self.replaceModule(originalId: swiftModuleId, replacementId: prebuiltModuleId, replacementInfo: newInfo, in: &modules) } else if let currentPrebuiltInfo = modules[prebuiltModuleId] { // Just update the isFramework bit on this prebuilt module dependency let newExternalModuleDetails = - try SwiftPrebuiltExternalModuleDetails(compiledModulePath: - TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), - isFramework: externalModuleDetails.isFramework) + SwiftPrebuiltExternalModuleDetails(compiledModulePath: + TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), + isFramework: externalModuleDetails.isFramework) let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), sourceFiles: [], directDependencies: currentPrebuiltInfo.directDependencies, + linkLibraries: currentPrebuiltInfo.linkLibraries, details: .swiftPrebuiltExternal(newExternalModuleDetails)) Self.replaceModule(originalId: prebuiltModuleId, replacementId: prebuiltModuleId, replacementInfo: newInfo, in: &modules) @@ -56,16 +59,7 @@ import func TSCBasic.topologicalSort extension InterModuleDependencyGraph { var topologicalSorting: [ModuleDependencyId] { get throws { - try topologicalSort(Array(modules.keys), - successors: { - var dependencies: [ModuleDependencyId] = [] - let moduleInfo = try moduleInfo(of: $0) - dependencies.append(contentsOf: moduleInfo.directDependencies ?? []) - if case .swift(let swiftModuleDetails) = moduleInfo.details { - dependencies.append(contentsOf: swiftModuleDetails.swiftOverlayDependencies ?? []) - } - return dependencies - }) + try topologicalSort(Array(modules.keys), successors: { try Array(moduleInfo(of: $0).allDependencies) }) } } @@ -88,22 +82,9 @@ extension InterModuleDependencyGraph { } // Traverse the set of modules in reverse topological order, assimilating transitive closures for moduleId in topologicalIdList.reversed() { - let moduleInfo = try moduleInfo(of: moduleId) - for dependencyId in moduleInfo.directDependencies! { + for dependencyId in try moduleInfo(of: moduleId).allDependencies { transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!) } - // For Swift dependencies, their corresponding Swift Overlay dependencies - // and bridging header dependencies are equivalent to direct dependencies. - if case .swift(let swiftModuleDetails) = moduleInfo.details { - let swiftOverlayDependencies = swiftModuleDetails.swiftOverlayDependencies ?? [] - for dependencyId in swiftOverlayDependencies { - transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!) - } - let bridgingHeaderDependencies = swiftModuleDetails.bridgingHeaderDependencies ?? [] - for dependencyId in bridgingHeaderDependencies { - transitiveClosureMap[moduleId]!.formUnion(transitiveClosureMap[dependencyId]!) - } - } } // For ease of use down-the-line, remove the node's self from its set of reachable nodes for (key, _) in transitiveClosureMap { @@ -114,67 +95,6 @@ extension InterModuleDependencyGraph { } @_spi(Testing) public extension InterModuleDependencyGraph { - /// Merge a module with a given ID and Info into a ModuleInfoMap - static func mergeModule(_ moduleId: ModuleDependencyId, - _ moduleInfo: ModuleInfo, - into moduleInfoMap: inout ModuleInfoMap) throws { - switch moduleId { - case .swift: - let prebuiltExternalModuleEquivalentId = - ModuleDependencyId.swiftPrebuiltExternal(moduleId.moduleName) - let placeholderEquivalentId = - ModuleDependencyId.swiftPlaceholder(moduleId.moduleName) - if moduleInfoMap[prebuiltExternalModuleEquivalentId] != nil || - moduleInfoMap[moduleId] != nil { - // If the set of discovered externalModules contains a .swiftPrebuiltExternal or .swift module - // with the same name, do not replace it. - break - } else if moduleInfoMap[placeholderEquivalentId] != nil { - // Replace the placeholder module with a full .swift ModuleInfo - // and fixup other externalModules' dependencies - replaceModule(originalId: placeholderEquivalentId, replacementId: moduleId, - replacementInfo: moduleInfo, in: &moduleInfoMap) - } else { - // Insert the new module - moduleInfoMap[moduleId] = moduleInfo - } - - case .swiftPrebuiltExternal: - // If the set of discovered externalModules contains a .swift module with the same name, - // replace it with the prebuilt version and fixup other externalModules' dependencies - let swiftModuleEquivalentId = ModuleDependencyId.swift(moduleId.moduleName) - let swiftPlaceholderEquivalentId = ModuleDependencyId.swiftPlaceholder(moduleId.moduleName) - if moduleInfoMap[swiftModuleEquivalentId] != nil { - // If the ModuleInfoMap contains an equivalent .swift module, replace it with the prebuilt - // version and update all other externalModules' dependencies - replaceModule(originalId: swiftModuleEquivalentId, replacementId: moduleId, - replacementInfo: moduleInfo, in: &moduleInfoMap) - } else if moduleInfoMap[swiftPlaceholderEquivalentId] != nil { - // If the moduleInfoMap contains an equivalent .swiftPlaceholder module, replace it with - // the prebuilt version and update all other externalModules' dependencies - replaceModule(originalId: swiftPlaceholderEquivalentId, replacementId: moduleId, - replacementInfo: moduleInfo, in: &moduleInfoMap) - } else { - // Insert the new module - moduleInfoMap[moduleId] = moduleInfo - } - - case .clang: - guard let existingModuleInfo = moduleInfoMap[moduleId] else { - moduleInfoMap[moduleId] = moduleInfo - break - } - // If this module *has* been seen before, merge the module infos to capture - // the super-set of so-far discovered dependencies of this module at various - // PCMArg scanning actions. - let combinedDependenciesInfo = mergeClangModuleInfoDependencies(moduleInfo, - existingModuleInfo) - replaceModule(originalId: moduleId, replacementId: moduleId, - replacementInfo: combinedDependenciesInfo, in: &moduleInfoMap) - case .swiftPlaceholder: - fatalError("Unresolved placeholder dependency at graph merge operation: \(moduleId)") - } - } /// Replace an existing module in the moduleInfoMap static func replaceModule(originalId: ModuleDependencyId, replacementId: ModuleDependencyId, @@ -204,93 +124,311 @@ extension InterModuleDependencyGraph { } } } +} - /// Given two moduleInfos of clang externalModules, merge them by combining their directDependencies and - /// dependenciesCapturedPCMArgs and sourceFiles fields. These fields may differ across the same module - /// scanned at different PCMArgs (e.g. -target option). - static func mergeClangModuleInfoDependencies(_ firstInfo: ModuleInfo, _ secondInfo:ModuleInfo - ) -> ModuleInfo { - guard case .clang(let firstDetails) = firstInfo.details, - case .clang(let secondDetails) = secondInfo.details - else { - fatalError("mergeClangModules expected two valid ClangModuleDetails objects.") +/// Incremental Build Machinery +internal extension InterModuleDependencyGraph { + /// We must determine if any of the module dependencies require re-compilation + /// Since we know that a prior dependency graph was not completely up-to-date, + /// there must be at least *some* dependencies that require being re-built. + /// + /// If a dependency is deemed as requiring a re-build, then every module + /// between it and the root (source module being built by this driver + /// instance) must also be re-built. + func computeInvalidatedModuleDependencies(fileSystem: FileSystem, + cas: SwiftScanCAS?, + forRebuild: Bool, + reporter: IncrementalCompilationState.Reporter? = nil) + throws -> Set { + let mainModuleInfo = mainModule + var modulesRequiringRebuild: Set = [] + var visited: Set = [] + // Scan from the main module's dependencies to avoid reporting + // the main module itself in the results. + for dependencyId in mainModuleInfo.allDependencies { + try outOfDateModuleScan(from: dependencyId, visited: &visited, + modulesRequiringRebuild: &modulesRequiringRebuild, + fileSystem: fileSystem, cas: cas, forRebuild: forRebuild, + reporter: reporter) } - // As far as their dependencies go, these module infos are identical - if firstInfo.directDependencies == secondInfo.directDependencies, - firstDetails.capturedPCMArgs == secondDetails.capturedPCMArgs, - firstInfo.sourceFiles == secondInfo.sourceFiles { - return firstInfo + if forRebuild && !modulesRequiringRebuild.isEmpty { + reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) } + return modulesRequiringRebuild + } - // Create a new moduleInfo that represents this module with combined dependency information - let firstModuleSources = firstInfo.sourceFiles ?? [] - let secondModuleSources = secondInfo.sourceFiles ?? [] - let combinedSourceFiles = Array(Set(firstModuleSources + secondModuleSources)) + /// From a set of provided module dependency pre-compilation jobs, + /// filter out those with a fully up-to-date output + func filterMandatoryModuleDependencyCompileJobs(_ allJobs: [Job], + fileSystem: FileSystem, + cas: SwiftScanCAS?, + reporter: IncrementalCompilationState.Reporter? = nil) throws -> [Job] { + // Determine which module pre-build jobs must be re-run + let modulesRequiringReBuild = + try computeInvalidatedModuleDependencies(fileSystem: fileSystem, cas: cas, forRebuild: true, reporter: reporter) - let firstModuleDependencies = firstInfo.directDependencies ?? [] - let secondModuleDependencies = secondInfo.directDependencies ?? [] - let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies)) + // Filter the `.generatePCM` and `.compileModuleFromInterface` jobs for + // modules which do *not* need re-building. + return allJobs.filter { job in + switch job.kind { + case .generatePCM: + return modulesRequiringReBuild.contains(.clang(job.moduleName)) + case .compileModuleFromInterface: + return modulesRequiringReBuild.contains(.swift(job.moduleName)) + default: + return true + } + } + } - let firstModuleCapturedPCMArgs = firstDetails.capturedPCMArgs ?? Set<[String]>() - let secondModuleCapturedPCMArgs = secondDetails.capturedPCMArgs ?? Set<[String]>() - let combinedCapturedPCMArgs = firstModuleCapturedPCMArgs.union(secondModuleCapturedPCMArgs) + /// Perform a postorder DFS to locate modules which are out-of-date with respect + /// to their inputs. Upon encountering such a module, add it to the set of invalidated + /// modules, along with the path from the root to this module. + func outOfDateModuleScan(from sourceModuleId: ModuleDependencyId, + visited: inout Set, + modulesRequiringRebuild: inout Set, + fileSystem: FileSystem, + cas: SwiftScanCAS?, + forRebuild: Bool, + reporter: IncrementalCompilationState.Reporter? = nil) throws { + let reportOutOfDate = { (name: String, reason: String) in + if forRebuild { + reporter?.reportExplicitDependencyWillBeReBuilt(sourceModuleId.moduleNameForDiagnostic, reason: reason) + } else { + reporter?.reportPriorExplicitDependencyStale(sourceModuleId.moduleNameForDiagnostic, reason: reason) + } + } - let combinedModuleDetails = - ClangModuleDetails(moduleMapPath: firstDetails.moduleMapPath, - contextHash: firstDetails.contextHash, - commandLine: firstDetails.commandLine, - capturedPCMArgs: combinedCapturedPCMArgs) + let sourceModuleInfo = try moduleInfo(of: sourceModuleId) + // Visit the module's dependencies + var hasOutOfDateModuleDependency = false + for dependencyId in sourceModuleInfo.allDependencies { + // If we have not already visited this module, recurse. + if !visited.contains(dependencyId) { + try outOfDateModuleScan(from: dependencyId, visited: &visited, + modulesRequiringRebuild: &modulesRequiringRebuild, + fileSystem: fileSystem, cas: cas, forRebuild: forRebuild, + reporter: reporter) + } + // Even if we're not revisiting a dependency, we must check if it's already known to be out of date. + hasOutOfDateModuleDependency = hasOutOfDateModuleDependency || modulesRequiringRebuild.contains(dependencyId) + } + + if hasOutOfDateModuleDependency { + reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Invalidated by downstream dependency") + modulesRequiringRebuild.insert(sourceModuleId) + } else if try !verifyModuleDependencyUpToDate(moduleID: sourceModuleId, fileSystem: fileSystem, cas:cas, reporter: reporter) { + reportOutOfDate(sourceModuleId.moduleNameForDiagnostic, "Out-of-date") + modulesRequiringRebuild.insert(sourceModuleId) + } - return ModuleInfo(modulePath: firstInfo.modulePath, - sourceFiles: combinedSourceFiles, - directDependencies: combinedDependencies, - details: .clang(combinedModuleDetails)) + // Now that we've determined if this module must be rebuilt, mark it as visited. + visited.insert(sourceModuleId) + } + + func outputMissingFromCAS(moduleInfo: ModuleInfo, + cas: SwiftScanCAS?) throws -> Bool { + func casOutputMissing(_ key: String?) throws -> Bool { + // Caching not enabled. + guard let id = key, let cas = cas else { return false } + // Do a local query to see if the output exists. + let result = try cas.queryCacheKey(id, globally: false) + // Make sure all outputs are available in local CAS. + guard let outputs = result else { return true } + return !outputs.allSatisfy { $0.isMaterialized } + } + + switch moduleInfo.details { + case .swift(let swiftDetails): + return try casOutputMissing(swiftDetails.moduleCacheKey) + case .clang(let clangDetails): + return try casOutputMissing(clangDetails.moduleCacheKey) + case .swiftPrebuiltExternal(_): + return false; + case .swiftPlaceholder(_): + // TODO: This should never ever happen. Hard error? + return true; + } + } + + func verifyModuleDependencyUpToDate(moduleID: ModuleDependencyId, + fileSystem: FileSystem, + cas: SwiftScanCAS?, + reporter: IncrementalCompilationState.Reporter?) throws -> Bool { + let checkedModuleInfo = try moduleInfo(of: moduleID) + // Check if there is a module cache key available, then the content that pointed by the cache key must + // exist for module to be up-to-date. Treat any CAS error as missing. + let missingFromCAS = (try? outputMissingFromCAS(moduleInfo: checkedModuleInfo, cas: cas)) ?? true + if missingFromCAS { + reporter?.reportExplicitDependencyMissingFromCAS(moduleID.moduleName) + return false + } + + // Verify that the specified input exists and is older than the specified output + let verifyInputOlderThanOutputModTime: (String, VirtualPath, TimePoint) -> Bool = + { moduleName, inputPath, outputModTime in + guard let inputModTime = + try? fileSystem.lastModificationTime(for: inputPath) else { + reporter?.report("Unable to 'stat' \(inputPath.description)") + return false + } + if inputModTime > outputModTime { + reporter?.reportExplicitDependencyOutOfDate(moduleName, + inputPath: inputPath.description) + return false + } + return true + } + + // Check if the output file exists + guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(checkedModuleInfo.modulePath.path)) else { + reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") + return false + } + + // We do not verify the binary module itself being out-of-date if we do not have a textual + // interface it was built from, but we can safely treat it as up-to-date, particularly + // because if it is newer than any of the modules they depend on it, they will + // still get invalidated in the check below for whether a module has + // any dependencies newer than it. + if case .swiftPrebuiltExternal(_) = moduleID { + return true + } + + // Check if a dependency of this module has a newer output than this module + for dependencyId in checkedModuleInfo.allDependencies { + let dependencyInfo = try moduleInfo(of: dependencyId) + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(dependencyInfo.modulePath.path), + outputModTime) { + return false + } + } + + // Check if any of the input sources of this module are newer than this module + switch checkedModuleInfo.details { + case .swift(let swiftDetails): + if let moduleInterfacePath = swiftDetails.moduleInterfacePath { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(moduleInterfacePath.path), + outputModTime) { + return false + } + } + if let bridgingHeaderPath = swiftDetails.bridgingHeaderPath { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(bridgingHeaderPath.path), + outputModTime) { + return false + } + } + for bridgingSourceFile in swiftDetails.bridgingSourceFiles ?? [] { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + VirtualPath.lookup(bridgingSourceFile.path), + outputModTime) { + return false + } + } + case .clang(_): + for inputSourceFile in checkedModuleInfo.sourceFiles ?? [] { + if !verifyInputOlderThanOutputModTime(moduleID.moduleName, + try VirtualPath(path: inputSourceFile), + outputModTime) { + return false + } + } + case .swiftPrebuiltExternal(_): + return true; + case .swiftPlaceholder(_): + // TODO: This should never ever happen. Hard error? + return false; + } + + return true } } internal extension InterModuleDependencyGraph { - func explainDependency(dependencyModuleName: String) throws -> [[ModuleDependencyId]]? { + func explainDependency(dependencyModuleName: String, allPaths: Bool) throws -> [[ModuleDependencyId]]? { guard modules.contains(where: { $0.key.moduleName == dependencyModuleName }) else { return nil } - var results = [[ModuleDependencyId]]() - try findAllPaths(source: .swift(mainModuleName), - to: dependencyModuleName, - pathSoFar: [.swift(mainModuleName)], - results: &results) - return Array(results) + var result: Set<[ModuleDependencyId]> = [] + if allPaths { + try findAllPaths(source: .swift(mainModuleName), + pathSoFar: [.swift(mainModuleName)], + results: &result, + destinationMatch: { $0.moduleName == dependencyModuleName }) + } else { + var visited: Set = [] + var singlePathResult: [ModuleDependencyId]? = nil + if try findAPath(source: .swift(mainModuleName), + pathSoFar: [.swift(mainModuleName)], + visited: &visited, + result: &singlePathResult, + destinationMatch: { $0.moduleName == dependencyModuleName }), + let resultingPath = singlePathResult { + result = [resultingPath] + } + } + return Array(result) } + @discardableResult + func findAPath(source: ModuleDependencyId, + pathSoFar: [ModuleDependencyId], + visited: inout Set, + result: inout [ModuleDependencyId]?, + destinationMatch: (ModuleDependencyId) -> Bool) throws -> Bool { + // Mark this node as visited + visited.insert(source) + let sourceInfo = try moduleInfo(of: source) + if destinationMatch(source) { + // If the source is a target Swift module, also check if it + // depends on a corresponding Clang module with the same name. + // If it does, add it to the path as well. + var completePath = pathSoFar + if sourceInfo.allDependencies.contains(.clang(source.moduleName)) { + completePath.append(.clang(source.moduleName)) + } + result = completePath + return true + } + + for dependency in sourceInfo.allDependencies { + if !visited.contains(dependency), + try findAPath(source: dependency, + pathSoFar: pathSoFar + [dependency], + visited: &visited, + result: &result, + destinationMatch: destinationMatch) { + return true + } + } + return false + } private func findAllPaths(source: ModuleDependencyId, - to moduleName: String, pathSoFar: [ModuleDependencyId], - results: inout [[ModuleDependencyId]]) throws { + results: inout Set<[ModuleDependencyId]>, + destinationMatch: (ModuleDependencyId) -> Bool) throws { let sourceInfo = try moduleInfo(of: source) - // If the source is our target, we are done - guard source.moduleName != moduleName else { + if destinationMatch(source) { // If the source is a target Swift module, also check if it // depends on a corresponding Clang module with the same name. // If it does, add it to the path as well. var completePath = pathSoFar - if let dependencies = sourceInfo.directDependencies, - dependencies.contains(.clang(moduleName)) { - completePath.append(.clang(moduleName)) + if sourceInfo.allDependencies.contains(.clang(source.moduleName)) { + completePath.append(.clang(source.moduleName)) } - results.append(completePath) + results.insert(completePath) return } - var allDependencies = sourceInfo.directDependencies ?? [] - if case .swift(let swiftModuleDetails) = sourceInfo.details, - let overlayDependencies = swiftModuleDetails.swiftOverlayDependencies { - allDependencies.append(contentsOf: overlayDependencies) - } - - for dependency in allDependencies { + for dependency in sourceInfo.allDependencies { try findAllPaths(source: dependency, - to: moduleName, pathSoFar: pathSoFar + [dependency], - results: &results) + results: &results, + destinationMatch: destinationMatch) } } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift index ffb83cae0..a7272b790 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift @@ -86,7 +86,7 @@ extension ModuleDependencyId: Codable { } /// Bridging header -public struct BridgingHeader: Codable { +public struct BridgingHeader: Codable, Hashable { var path: TextualVirtualPath /// The source files referenced by the bridging header. var sourceFiles: [TextualVirtualPath] @@ -94,8 +94,15 @@ public struct BridgingHeader: Codable { var moduleDependencies: [String] } +/// Linked Library +public struct LinkLibraryInfo: Codable, Hashable { + public var linkName: String + public var isFramework: Bool + public var shouldForceLoad: Bool +} + /// Details specific to Swift modules. -public struct SwiftModuleDetails: Codable { +public struct SwiftModuleDetails: Codable, Hashable { /// The module interface from which this module was built, if any. public var moduleInterfacePath: TextualVirtualPath? @@ -117,7 +124,7 @@ public struct SwiftModuleDetails: Codable { /// Options to the compile command public var commandLine: [String]? = [] - /// Options to the compile command + /// Options to the compile bridging header command public var bridgingPchCommandLine: [String]? = [] /// The context hash for this module that encodes the producing interface's path, @@ -125,23 +132,26 @@ public struct SwiftModuleDetails: Codable { /// corresponding to the main module being built. public var contextHash: String? - /// To build a PCM to be used by this Swift module, we need to append these - /// arguments to the generic PCM build arguments reported from the dependency - /// graph. - public var extraPcmArgs: [String] - /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? /// A set of Swift Overlays of Clang Module Dependencies - var swiftOverlayDependencies: [ModuleDependencyId]? + public var swiftOverlayDependencies: [ModuleDependencyId]? + + /// A set of directly-imported in source module dependencies + public var sourceImportDependencies: [ModuleDependencyId]? /// The module cache key of the output module. public var moduleCacheKey: String? + + /// Chained bridging header path + public var chainedBridgingHeaderPath: String? + /// Chained bridging header content + public var chainedBridgingHeaderContent: String? } /// Details specific to Swift placeholder dependencies. -public struct SwiftPlaceholderModuleDetails: Codable { +public struct SwiftPlaceholderModuleDetails: Codable, Hashable { /// The path to the .swiftModuleDoc file. var moduleDocPath: TextualVirtualPath? @@ -150,7 +160,7 @@ public struct SwiftPlaceholderModuleDetails: Codable { } /// Details specific to Swift externally-pre-built modules. -public struct SwiftPrebuiltExternalModuleDetails: Codable { +public struct SwiftPrebuiltExternalModuleDetails: Codable, Hashable { /// The path to the already-compiled module that must be used instead of /// generating a job to build this module. public var compiledModulePath: TextualVirtualPath @@ -164,28 +174,18 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable { /// The paths to the binary module's header dependencies public var headerDependencyPaths: [TextualVirtualPath]? + /// Clang module dependencies of the textual header input + public var headerDependencyModuleDependencies: [ModuleDependencyId]? + /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? /// The module cache key of the pre-built module. public var moduleCacheKey: String? - - public init(compiledModulePath: TextualVirtualPath, - moduleDocPath: TextualVirtualPath? = nil, - moduleSourceInfoPath: TextualVirtualPath? = nil, - headerDependencies: [TextualVirtualPath]? = nil, - isFramework: Bool, moduleCacheKey: String? = nil) throws { - self.compiledModulePath = compiledModulePath - self.moduleDocPath = moduleDocPath - self.moduleSourceInfoPath = moduleSourceInfoPath - self.headerDependencyPaths = headerDependencies - self.isFramework = isFramework - self.moduleCacheKey = moduleCacheKey - } } /// Details specific to Clang modules. -public struct ClangModuleDetails: Codable { +public struct ClangModuleDetails: Codable, Hashable { /// The path to the module map used to build this module. public var moduleMapPath: TextualVirtualPath @@ -195,41 +195,31 @@ public struct ClangModuleDetails: Codable { /// Options to the compile command public var commandLine: [String] = [] - /// Set of PCM Arguments of depending modules which - /// are covered by the directDependencies info of this module - public var capturedPCMArgs: Set<[String]>? - /// The module cache key of the output module. public var moduleCacheKey: String? - - public init(moduleMapPath: TextualVirtualPath, - contextHash: String, - commandLine: [String], - capturedPCMArgs: Set<[String]>?, - moduleCacheKey: String? = nil) { - self.moduleMapPath = moduleMapPath - self.contextHash = contextHash - self.commandLine = commandLine - self.capturedPCMArgs = capturedPCMArgs - self.moduleCacheKey = moduleCacheKey - } } -public struct ModuleInfo: Codable { +public struct ModuleInfo: Codable, Hashable { /// The path for the module. public var modulePath: TextualVirtualPath /// The source files used to build this module. public var sourceFiles: [String]? - /// The set of direct module dependencies of this module. + /// The set of directly-imported module dependencies of this module. + /// For the complete set of all module dependencies of this module, + /// including Swift overlay dependencies and bridging header dependenceis, + /// use the `allDependencies` computed property. public var directDependencies: [ModuleDependencyId]? + /// The set of libraries that need to be linked + public var linkLibraries: [LinkLibraryInfo]? + /// Specific details of a particular kind of module. public var details: Details /// Specific details of a particular kind of module. - public enum Details { + public enum Details: Hashable { /// Swift modules may be built from a module interface, and may have /// a bridging header. case swift(SwiftModuleDetails) @@ -248,10 +238,12 @@ public struct ModuleInfo: Codable { public init(modulePath: TextualVirtualPath, sourceFiles: [String]?, directDependencies: [ModuleDependencyId]?, + linkLibraries: [LinkLibraryInfo]?, details: Details) { self.modulePath = modulePath self.sourceFiles = sourceFiles self.directDependencies = directDependencies + self.linkLibraries = linkLibraries self.details = details } } @@ -302,6 +294,48 @@ extension ModuleInfo.Details: Codable { } } +extension ModuleInfo { + var bridgingHeaderModuleDependencies: [ModuleDependencyId]? { + switch details { + case .swift(let swiftDetails): + return swiftDetails.bridgingHeaderDependencies + case .swiftPrebuiltExternal(let swiftPrebuiltDetails): + return swiftPrebuiltDetails.headerDependencyModuleDependencies + default: + return nil + } + } +} + +public extension ModuleInfo { + // Directly-imported dependencies plus additional dependency + // kinds for Swift modules: + // - Swift overlay dependencies + // - Bridging Header dependencies + var allDependencies: [ModuleDependencyId] { + var result: [ModuleDependencyId] = directDependencies ?? [] + if case .swift(let swiftModuleDetails) = details { + // Ensure the dependnecies emitted are unique and follow a predictable ordering: + // 1. directDependencies in the order reported by the scanner + // 2. swift overlay dependencies + // 3. briding header dependencies + var addedSoFar: Set = [] + addedSoFar.formUnion(directDependencies ?? []) + for depId in swiftModuleDetails.swiftOverlayDependencies ?? [] { + if addedSoFar.insert(depId).inserted { + result.append(depId) + } + } + for depId in swiftModuleDetails.bridgingHeaderDependencies ?? [] { + if addedSoFar.insert(depId).inserted { + result.append(depId) + } + } + } + return result + } +} + /// Describes the complete set of dependencies for a Swift module, including /// all of the Swift and C modules and source files it depends on. public struct InterModuleDependencyGraph: Codable { @@ -315,7 +349,7 @@ public struct InterModuleDependencyGraph: Codable { public var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! } } -internal extension InterModuleDependencyGraph { +@_spi(Testing) public extension InterModuleDependencyGraph { func toJSONData() throws -> Data { let encoder = JSONEncoder() #if os(Linux) || os(Android) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 6065ab71f..06e42cabf 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -13,6 +13,7 @@ import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath import struct Foundation.Data +import var TSCBasic.localFileSystem import Dispatch @@ -45,118 +46,112 @@ public class InterModuleDependencyOracle { @_spi(Testing) public func getDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]? = nil, - commandLine: [String]) + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { precondition(hasScannerInstance) return try swiftScanLibInstance!.scanDependencies(workingDirectory: workingDirectory, moduleAliases: moduleAliases, - invocationCommand: commandLine) - } - - @_spi(Testing) public func getBatchDependencies(workingDirectory: AbsolutePath, - moduleAliases: [String: String]? = nil, - commandLine: [String], - batchInfos: [BatchScanModuleInfo]) - throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { - precondition(hasScannerInstance) - return try swiftScanLibInstance!.batchScanDependencies(workingDirectory: workingDirectory, - moduleAliases: moduleAliases, - invocationCommand: commandLine, - batchInfos: batchInfos) + invocationCommand: commandLine, + diagnostics: &diagnostics) } @_spi(Testing) public func getImports(workingDirectory: AbsolutePath, moduleAliases: [String: String]? = nil, - commandLine: [String]) + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { precondition(hasScannerInstance) return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory, moduleAliases: moduleAliases, - invocationCommand: commandLine) + invocationCommand: commandLine, + diagnostics: &diagnostics) + } + + @available(*, deprecated, message: "use verifyOrCreateScannerInstance(swiftScanLibPath:)") + public func verifyOrCreateScannerInstance(fileSystem: FileSystem, + swiftScanLibPath: AbsolutePath) throws { + return try verifyOrCreateScannerInstance(swiftScanLibPath: swiftScanLibPath) } /// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library - /// Returns True if a library instance exists (either verified or newly-created). - @_spi(Testing) public func verifyOrCreateScannerInstance(fileSystem: FileSystem, - swiftScanLibPath: AbsolutePath) - throws -> Bool { + public func verifyOrCreateScannerInstance(swiftScanLibPath: AbsolutePath?) throws { return try queue.sync { - if swiftScanLibInstance == nil { - guard fileSystem.exists(swiftScanLibPath) else { - return false - } + guard let scanInstance = swiftScanLibInstance else { swiftScanLibInstance = try SwiftScan(dylib: swiftScanLibPath) - } else { - guard swiftScanLibInstance!.path == swiftScanLibPath else { - throw DependencyScanningError - .scanningLibraryInvocationMismatch(swiftScanLibInstance!.path, swiftScanLibPath) - } + return + } + + guard scanInstance.path?.description == swiftScanLibPath?.description else { + throw DependencyScanningError + .scanningLibraryInvocationMismatch(scanInstance.path?.description ?? "built-in", + swiftScanLibPath?.description ?? "built-in") } - return true } } - @_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) { + @_spi(Testing) public func supportsBinaryFrameworkDependencies() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to serialize scanner cache with no scanner instance.") - } - if swiftScan.canLoadStoreScannerCache { - swiftScan.serializeScannerCache(to: path) + fatalError("Attempting to query supported scanner API with no scanner instance.") } + return swiftScan.hasBinarySwiftModuleIsFramework } - @_spi(Testing) public func loadScannerCache(from path: AbsolutePath) -> Bool { + @_spi(Testing) public func supportsBinaryModuleHeaderModuleDependencies() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to load scanner cache with no scanner instance.") - } - if swiftScan.canLoadStoreScannerCache { - return swiftScan.loadScannerCache(from: path) + fatalError("Attempting to query supported scanner API with no scanner instance.") } - return false + return swiftScan.hasBinarySwiftModuleHeaderModuleDependencies } - @_spi(Testing) public func resetScannerCache() { + @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to reset scanner cache with no scanner instance.") - } - if swiftScan.canLoadStoreScannerCache { - swiftScan.resetScannerCache() + fatalError("Attempting to query supported scanner API with no scanner instance.") } + return swiftScan.supportsScannerDiagnostics } - @_spi(Testing) public func supportsBinaryFrameworkDependencies() throws -> Bool { + @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.hasBinarySwiftModuleIsFramework + return swiftScan.supportsBinaryModuleHeaderDependencies || swiftScan.supportsBinaryModuleHeaderDependency } - @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool { + @_spi(Testing) public var supportsBridgingHeaderPCHCommand: Bool { + guard let swiftScan = swiftScanLibInstance else { + // If no scanner, feature is not supported. + return false + } + return swiftScan.supportsBridgingHeaderPCHCommand + } + + @_spi(Testing) public func supportsPerScanDiagnostics() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsScannerDiagnostics + return swiftScan.canQueryPerScanDiagnostics } - @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool { + @_spi(Testing) public func supportsDiagnosticSourceLocations() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsBinaryModuleHeaderDependencies + return swiftScan.supportsDiagnosticSourceLocations } - @_spi(Testing) public func supportsCaching() throws -> Bool { + @_spi(Testing) public func supportsLinkLibraries() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsCaching + return swiftScan.supportsLinkLibraries } - @_spi(Testing) public func supportsBridgingHeaderPCHCommand() throws -> Bool { + @_spi(Testing) public func supportsSeparateImportOnlyDependencise() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsBridgingHeaderPCHCommand + return swiftScan.supportsSeparateImportOnlyDependencise } @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { @@ -171,36 +166,64 @@ public class InterModuleDependencyOracle { return diags.isEmpty ? nil : diags } - public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws { + public func getOrCreateCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } - try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions) - } - - public func store(data: Data) throws -> String { - guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to reset scanner cache with no scanner instance.") - } - return try swiftScan.store(data:data) - } - - public func computeCacheKeyForOutput(kind: FileType, commandLine: [Job.ArgTemplate], input: VirtualPath.Handle?) throws -> String { - guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to reset scanner cache with no scanner instance.") + // Use synchronized queue to avoid creating multiple OnDisk CAS at the same location as that will leave to synchronization issues. + return try queue.sync { + let casOpt = CASConfig(onDiskPath: onDiskPath, pluginPath: pluginPath, pluginOptions: pluginOptions) + if let cas = createdCASMap[casOpt] { + return cas + } + let cas = try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions) + createdCASMap[casOpt] = cas + return cas } - let inputPath = input?.description ?? "" - return try swiftScan.computeCacheKeyForOutput(kind: kind, commandLine: commandLine.stringArray, input: inputPath) } + // Note: this is `true` even in the `compilerIntegratedTooling` mode + // where the `SwiftScan` instance refers to the own image the driver is + // running in, since there is still technically a `SwiftScan` handle + // capable of handling API requests expected of it. private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil } + func getScannerInstance() -> SwiftScan? { + self.swiftScanLibInstance + } + func setScannerInstance(_ instance: SwiftScan?) { + self.swiftScanLibInstance = instance + } /// Queue to sunchronize accesses to the scanner - internal let queue = DispatchQueue(label: "org.swift.swift-driver.swift-scan") + let queue = DispatchQueue(label: "org.swift.swift-driver.swift-scan") /// A reference to an instance of the compiler's libSwiftScan shared library private var swiftScanLibInstance: SwiftScan? = nil internal let scannerRequiresPlaceholderModules: Bool + + internal struct CASConfig: Hashable, Equatable { + static func == (lhs: InterModuleDependencyOracle.CASConfig, rhs: InterModuleDependencyOracle.CASConfig) -> Bool { + return lhs.onDiskPath == rhs.onDiskPath && + lhs.pluginPath == rhs.pluginPath && + lhs.pluginOptions.elementsEqual(rhs.pluginOptions, by: ==) + } + + func hash(into hasher: inout Hasher) { + hasher.combine(onDiskPath) + hasher.combine(pluginPath) + for opt in pluginOptions { + hasher.combine(opt.0) + hasher.combine(opt.1) + } + } + + let onDiskPath: AbsolutePath? + let pluginPath: AbsolutePath? + let pluginOptions: [(String, String)] + } + + /// Storing the CAS created via CASConfig. + internal var createdCASMap: [CASConfig: SwiftScanCAS] = [:] } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index a12c1e14f..d6f488f62 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// -import TSCBasic // <<< import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath +import struct TSCBasic.RelativePath import struct TSCBasic.Diagnostic import var TSCBasic.localFileSystem import var TSCBasic.stdoutStream @@ -24,11 +24,17 @@ import class Foundation.JSONDecoder import var Foundation.EXIT_SUCCESS extension Diagnostic.Message { - static func warn_scanner_frontend_fallback() -> Diagnostic.Message { - .warning("Fallback to `swift-frontend` dependency scanner invocation") + static func warn_scan_dylib_not_found() -> Diagnostic.Message { + .warning("Unable to locate libSwiftScan. Fallback to `swift-frontend` dependency scanner invocation.") + } + static func warn_scan_dylib_load_failed(_ libPath: String) -> Diagnostic.Message { + .warning("In-process dependency scan query failed due to incompatible libSwiftScan (\(libPath)). Fallback to `swift-frontend` dependency scanner invocation. Specify '-nonlib-dependency-scanner' to silence this warning.") + } + static func error_caching_enabled_libswiftscan_load_failure(_ libPath: String) -> Diagnostic.Message { + .error("Swift Caching enabled - libSwiftScan load failed (\(libPath)).") } static func scanner_diagnostic_error(_ message: String) -> Diagnostic.Message { - .error("Dependency scanning failure: \(message)") + return .error(message) } static func scanner_diagnostic_warn(_ message: String) -> Diagnostic.Message { .warning(message) @@ -49,7 +55,7 @@ extension Diagnostic.Message { var dependencyGraph = try performDependencyScan() if parsedOptions.hasArgument(.printPreprocessedExplicitDependencyGraph) { - try stdoutStream <<< dependencyGraph.toJSONString() + try stdoutStream.send(dependencyGraph.toJSONString()) stdoutStream.flush() } @@ -68,7 +74,7 @@ extension Diagnostic.Message { if parsedOptions.hasArgument(.printExplicitDependencyGraph) { let outputFormat = parsedOptions.getLastArgument(.explicitDependencyGraphFormat)?.asSingle if outputFormat == nil || outputFormat == "json" { - try stdoutStream <<< dependencyGraph.toJSONString() + try stdoutStream.send(dependencyGraph.toJSONString()) } else if outputFormat == "dot" { DOTModuleDependencyGraphSerializer(dependencyGraph).writeDOT(to: &stdoutStream) } @@ -109,7 +115,7 @@ public extension Driver { try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) - // FIXME: MSVC runtime flags + try addRuntimeLibraryFlags(commandLine: &commandLine) // Pass in external target dependencies to be treated as placeholder dependencies by the scanner if let externalTargetDetailsMap = externalTargetModuleDetailsMap, @@ -120,10 +126,78 @@ public extension Driver { commandLine.appendPath(dependencyPlaceholderMapFile) } - try commandLine.appendLast(.clangIncludeTree, from: &parsedOptions) if isFrontendArgSupported(.clangScannerModuleCachePath) { try commandLine.appendLast(.clangScannerModuleCachePath, from: &parsedOptions) } + if isFrontendArgSupported(.sdkModuleCachePath) { + try commandLine.appendLast(.sdkModuleCachePath, from: &parsedOptions) + } + + if isFrontendArgSupported(.scannerModuleValidation) { + commandLine.appendFlag(.scannerModuleValidation) + } + + if isFrontendArgSupported(.scannerPrefixMap) { + // construct `-scanner-prefix-mapper` for scanner. + for (key, value) in prefixMapping { + commandLine.appendFlag(.scannerPrefixMap) + commandLine.appendFlag(key.pathString + "=" + value.pathString) + } + } + + if (parsedOptions.contains(.driverShowIncremental) || + parsedOptions.contains(.dependencyScanCacheRemarks)) && + isFrontendArgSupported(.dependencyScanCacheRemarks) { + commandLine.appendFlag(.dependencyScanCacheRemarks) + } + + if shouldAttemptIncrementalCompilation && + parsedOptions.contains(.incrementalDependencyScan) { + if let serializationPath = buildRecordInfo?.dependencyScanSerializedResultPath { + if isFrontendArgSupported(.validatePriorDependencyScanCache) { + // Any compiler which supports "-validate-prior-dependency-scan-cache" + // also supports "-load-dependency-scan-cache" + // and "-serialize-dependency-scan-cache" and "-dependency-scan-cache-path" + commandLine.appendFlag(.dependencyScanCachePath) + commandLine.appendPath(serializationPath) + commandLine.appendFlag(.reuseDependencyScanCache) + commandLine.appendFlag(.validatePriorDependencyScanCache) + commandLine.appendFlag(.serializeDependencyScanCache) + } + } + } + + if isFrontendArgSupported(.autoBridgingHeaderChaining) { + if parsedOptions.hasFlag(positive: .autoBridgingHeaderChaining, + negative: .noAutoBridgingHeaderChaining, + default: false) || isCachingEnabled { + if producePCHJob { + commandLine.appendFlag(.autoBridgingHeaderChaining) + } else { + diagnosticEngine.emit(.warning("-auto-bridging-header-chaining requires generatePCH job, no chaining will be performed")) + commandLine.appendFlag(.noAutoBridgingHeaderChaining) + } + } else { + commandLine.appendFlag(.noAutoBridgingHeaderChaining) + } + } + + // Provide a directory to path to scanner for where the chained bridging header will be. + // Prefer writing next to pch output, otherwise next to module output path before fallback to temp directory for non-caching build. + if isFrontendArgSupported(.scannerOutputDir) { + if let outputDir = try? computePrecompiledBridgingHeaderDir(&parsedOptions, + compilerMode: compilerMode) { + commandLine.appendFlag(.scannerOutputDir) + commandLine.appendPath(outputDir) + } else { + commandLine.appendFlag(.scannerOutputDir) + commandLine.appendPath(VirtualPath.temporary(try RelativePath(validating: "scanner"))) + } + } + + if isFrontendArgSupported(.resolvedPluginVerification) { + commandLine.appendFlag(.resolvedPluginVerification) + } // Pass on the input files commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) }) @@ -144,34 +218,8 @@ public extension Driver { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted] let contents = try encoder.encode(placeholderArtifacts) - return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleOutputInfo.name)-external-modules.json"), - contents) - } - - /// Returns false if the lib is available and ready to use - private mutating func initSwiftScanLib() throws -> Bool { - // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found, - // attempt to fallback to using `swift-frontend -scan-dependencies` invocations for dependency - // scanning. - var fallbackToFrontend = parsedOptions.hasArgument(.driverScanDependenciesNonLib) - let optionalScanLibPath = try toolchain.lookupSwiftScanLib() - if let scanLibPath = optionalScanLibPath, - try interModuleDependencyOracle - .verifyOrCreateScannerInstance(fileSystem: fileSystem, - swiftScanLibPath: scanLibPath) == false { - fallbackToFrontend = true - // This warning is mostly useful for debugging the driver, so let's hide it - // when libSwiftDriver is used, instead of a swift-driver executable. - if !integratedDriver { - diagnosticEngine.emit(.warn_scanner_frontend_fallback()) - } - } - if !fallbackToFrontend && enableCaching { - try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(), - onDiskPath: try getOnDiskCASPath(), - pluginOptions: try getCASPluginOptions()) - } - return fallbackToFrontend + return try VirtualPath.createUniqueTemporaryFileWithKnownContents(.init(validating: "\(moduleOutputInfo.name)-external-modules.json"), + contents) } static func sanitizeCommandForLibScanInvocation(_ command: inout [String]) { @@ -192,18 +240,25 @@ public extension Driver { let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles) let imports: InterModuleDependencyImports - let isSwiftScanLibAvailable = !(try initSwiftScanLib()) - if isSwiftScanLibAvailable { - let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! + if supportInProcessSwiftScanQueries { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] + guard let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory else { + throw DependencyScanningError.dependencyScanFailed("cannot determine working directory") + } var command = try Self.itemizedJobCommand(of: preScanJob, useResponseFiles: .disabled, using: executor.resolver) Self.sanitizeCommandForLibScanInvocation(&command) - imports = - try interModuleDependencyOracle.getImports(workingDirectory: cwd, - moduleAliases: moduleOutputInfo.aliases, - commandLine: command) - + do { + imports = try interModuleDependencyOracle.getImports(workingDirectory: cwd, + moduleAliases: moduleOutputInfo.aliases, + commandLine: command, + diagnostics: &scanDiagnostics) + } catch let DependencyScanningError.dependencyScanFailed(reason) { + try emitGlobalScannerDiagnostics() + throw DependencyScanningError.dependencyScanFailed(reason) + } + try emitGlobalScannerDiagnostics() } else { // Fallback to legacy invocation of the dependency scanner with // `swift-frontend -scan-dependencies -import-prescan` @@ -216,7 +271,40 @@ public extension Driver { return imports } - mutating internal func performDependencyScan() throws -> InterModuleDependencyGraph { + internal func emitScannerDiagnostics(_ diagnostics: [ScannerDiagnosticPayload]) throws { + for diagnostic in diagnostics { + switch diagnostic.severity { + case .error: + diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message), + location: diagnostic.sourceLocation) + case .warning: + diagnosticEngine.emit(.scanner_diagnostic_warn(diagnostic.message), + location: diagnostic.sourceLocation) + case .note: + diagnosticEngine.emit(.scanner_diagnostic_note(diagnostic.message), + location: diagnostic.sourceLocation) + case .remark: + diagnosticEngine.emit(.scanner_diagnostic_remark(diagnostic.message), + location: diagnostic.sourceLocation) + case .ignored: + diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message), + location: diagnostic.sourceLocation) + } + } + } + + mutating internal func emitGlobalScannerDiagnostics() throws { + // We only emit global scanner-collected diagnostics as a legacy flow + // when the scanner does not support per-scan diagnostic output + guard try !interModuleDependencyOracle.supportsPerScanDiagnostics() else { + return + } + if let diags = try interModuleDependencyOracle.getScannerDiagnostics() { + try emitScannerDiagnostics(diags) + } + } + + mutating func performDependencyScan() throws -> InterModuleDependencyGraph { let scannerJob = try dependencyScanningJob() let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles) let dependencyGraph: InterModuleDependencyGraph @@ -224,38 +312,30 @@ public extension Driver { if parsedOptions.contains(.v) { let arguments: [String] = try executor.resolver.resolveArgumentList(for: scannerJob, useResponseFiles: .disabled) - stdoutStream <<< arguments.map { $0.spm_shellEscaped() }.joined(separator: " ") <<< "\n" + stdoutStream.send("\(arguments.map { $0.spm_shellEscaped() }.joined(separator: " "))\n") stdoutStream.flush() } - let isSwiftScanLibAvailable = !(try initSwiftScanLib()) - if isSwiftScanLibAvailable { - let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! + if supportInProcessSwiftScanQueries { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] + guard let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory else { + throw DependencyScanningError.dependencyScanFailed("cannot determine working directory") + } var command = try Self.itemizedJobCommand(of: scannerJob, useResponseFiles: .disabled, using: executor.resolver) Self.sanitizeCommandForLibScanInvocation(&command) - dependencyGraph = - try interModuleDependencyOracle.getDependencies(workingDirectory: cwd, - moduleAliases: moduleOutputInfo.aliases, - commandLine: command) - let possibleDiags = try interModuleDependencyOracle.getScannerDiagnostics() - if let diags = possibleDiags { - for diagnostic in diags { - switch diagnostic.severity { - case .error: - diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message)) - case .warning: - diagnosticEngine.emit(.scanner_diagnostic_warn(diagnostic.message)) - case .note: - diagnosticEngine.emit(.scanner_diagnostic_note(diagnostic.message)) - case .remark: - diagnosticEngine.emit(.scanner_diagnostic_remark(diagnostic.message)) - case .ignored: - diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message)) - } - } + do { + dependencyGraph = try interModuleDependencyOracle.getDependencies(workingDirectory: cwd, + moduleAliases: moduleOutputInfo.aliases, + commandLine: command, + diagnostics: &scanDiagnostics) + try emitScannerDiagnostics(scanDiagnostics) + } catch let DependencyScanningError.dependencyScanFailed(reason) { + try emitGlobalScannerDiagnostics() + throw DependencyScanningError.dependencyScanFailed(reason) } + try emitGlobalScannerDiagnostics() } else { // Fallback to legacy invocation of the dependency scanner with // `swift-frontend -scan-dependencies` @@ -268,76 +348,6 @@ public extension Driver { return dependencyGraph } - mutating internal func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo]) - throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { - let batchScanningJob = try batchDependencyScanningJob(for: moduleInfos) - let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles) - let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] - - let isSwiftScanLibAvailable = !(try initSwiftScanLib()) - if isSwiftScanLibAvailable { - let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! - var command = try Self.itemizedJobCommand(of: batchScanningJob, - useResponseFiles: .disabled, - using: executor.resolver) - Self.sanitizeCommandForLibScanInvocation(&command) - moduleVersionedGraphMap = - try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd, - moduleAliases: moduleOutputInfo.aliases, - commandLine: command, - batchInfos: moduleInfos) - } else { - // Fallback to legacy invocation of the dependency scanner with - // `swift-frontend -scan-dependencies` - moduleVersionedGraphMap = try executeLegacyBatchScan(moduleInfos: moduleInfos, - batchScanningJob: batchScanningJob, - forceResponseFiles: forceResponseFiles) - } - return moduleVersionedGraphMap - } - - // Perform a batch scan by invoking the command-line dependency scanner and decoding the resulting - // JSON. - fileprivate func executeLegacyBatchScan(moduleInfos: [BatchScanModuleInfo], - batchScanningJob: Job, - forceResponseFiles: Bool) - throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { - let batchScanResult = - try self.executor.execute(job: batchScanningJob, - forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) - let success = batchScanResult.exitStatus == .terminated(code: EXIT_SUCCESS) - guard success else { - throw JobExecutionError.jobFailedWithNonzeroExitCode( - type(of: executor).computeReturnCode(exitStatus: batchScanResult.exitStatus), - try batchScanResult.utf8stderrOutput()) - } - // Decode the resulting dependency graphs and build a dictionary from a moduleId to - // a set of dependency graphs that were built for it - let moduleVersionedGraphMap = - try moduleInfos.reduce(into: [ModuleDependencyId: [InterModuleDependencyGraph]]()) { - let moduleId: ModuleDependencyId - let dependencyGraphPath: VirtualPath - switch $1 { - case .swift(let swiftModuleBatchScanInfo): - moduleId = .swift(swiftModuleBatchScanInfo.swiftModuleName) - dependencyGraphPath = try VirtualPath(path: swiftModuleBatchScanInfo.output) - case .clang(let clangModuleBatchScanInfo): - moduleId = .clang(clangModuleBatchScanInfo.clangModuleName) - dependencyGraphPath = try VirtualPath(path: clangModuleBatchScanInfo.output) - } - let contents = try fileSystem.readFileContents(dependencyGraphPath) - let decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self, - from: Data(contents.contents)) - if $0[moduleId] != nil { - $0[moduleId]!.append(decodedGraph) - } else { - $0[moduleId] = [decodedGraph] - } - } - return moduleVersionedGraphMap - } - /// Precompute the set of module names as imported by the current module mutating private func importPreScanningJob() throws -> Job { // Aggregate the fast dependency scanner arguments @@ -349,7 +359,7 @@ public extension Driver { try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, bridgingHeaderHandling: .parsed, moduleDependencyGraphUse: .dependencyScan) - // FIXME: MSVC runtime flags + try addRuntimeLibraryFlags(commandLine: &commandLine) // Pass on the input files commandLine.append(contentsOf: inputFiles.map { .path($0.file) }) @@ -365,64 +375,6 @@ public extension Driver { outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)]) } - /// Precompute the dependencies for a given collection of modules using swift frontend's batch scanning mode - mutating private func batchDependencyScanningJob(for moduleInfos: [BatchScanModuleInfo]) throws -> Job { - var inputs: [TypedVirtualPath] = [] - - // Aggregate the fast dependency scanner arguments - var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - - // The dependency scanner automatically operates in batch mode if -batch-scan-input-file - // is present. - commandLine.appendFlag("-frontend") - commandLine.appendFlag("-scan-dependencies") - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .scanDependencies, - bridgingHeaderHandling: .ignored, - moduleDependencyGraphUse: .dependencyScan) - - let batchScanInputFilePath = try serializeBatchScanningModuleArtifacts(moduleInfos: moduleInfos) - commandLine.appendFlag("-batch-scan-input-file") - commandLine.appendPath(batchScanInputFilePath) - - // This action does not require any input files, but all frontend actions require - // at least one input so pick any input of the current compilation. - let inputFile = inputFiles.first { $0.type == .swift } - commandLine.appendPath(inputFile!.file) - inputs.append(inputFile!) - - // This job's outputs are defined as a set of dependency graph json files - let outputs: [TypedVirtualPath] = try moduleInfos.map { - switch $0 { - case .swift(let swiftModuleBatchScanInfo): - return TypedVirtualPath(file: try VirtualPath.intern(path: swiftModuleBatchScanInfo.output), - type: .jsonDependencies) - case .clang(let clangModuleBatchScanInfo): - return TypedVirtualPath(file: try VirtualPath.intern(path: clangModuleBatchScanInfo.output), - type: .jsonDependencies) - } - } - - // Construct the scanning job. - return Job(moduleName: moduleOutputInfo.name, - kind: .scanDependencies, - tool: try toolchain.resolvedTool(.swiftCompiler), - commandLine: commandLine, - displayInputs: inputs, - inputs: inputs, - primaryInputs: [], - outputs: outputs) - } - - /// Serialize a collection of modules into an input format expected by the batch module dependency scanner. - func serializeBatchScanningModuleArtifacts(moduleInfos: [BatchScanModuleInfo]) - throws -> VirtualPath { - let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted] - let contents = try encoder.encode(moduleInfos) - return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleOutputInfo.name)-batch-module-scan.json"), - contents) - } - static func itemizedJobCommand(of job: Job, useResponseFiles: ResponseFileHandling, using resolver: ArgsResolver) throws -> [String] { // Because the command-line passed to libSwiftScan does not go through the shell @@ -439,3 +391,7 @@ public extension Driver { .parentDirectory // toolchain root } } + +extension Driver { + var supportInProcessSwiftScanQueries: Bool { return self.swiftScanLibInstance != nil } +} diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift index 5f6e51faa..4b62771a7 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift @@ -15,7 +15,7 @@ /// - Swift Module Path /// - Swift Doc Path /// - Swift Source Info Path -@_spi(Testing) public struct SwiftModuleArtifactInfo: Codable { +@_spi(Testing) public struct SwiftModuleArtifactInfo: Codable, Hashable { /// The module's name public let moduleName: String /// The path for the module's .swiftmodule file @@ -48,7 +48,7 @@ /// - Clang Module (name) /// - Clang Module (PCM) Path /// - Clang Module Map Path -@_spi(Testing) public struct ClangModuleArtifactInfo: Codable { +@_spi(Testing) public struct ClangModuleArtifactInfo: Codable, Hashable { /// The module's name public let moduleName: String /// The path for the module's .pcm file @@ -57,24 +57,28 @@ public let clangModuleMapPath: TextualVirtualPath /// A flag to indicate whether this module is a framework public let isFramework: Bool + /// A flag to indicate whether this module is a dependency + /// of the main module's bridging header + public let isBridgingHeaderDependency: Bool /// The cache key for the module. public let clangModuleCacheKey: String? init(name: String, modulePath: TextualVirtualPath, moduleMapPath: TextualVirtualPath, - moduleCacheKey: String? = nil) { + moduleCacheKey: String? = nil, isBridgingHeaderDependency: Bool = true) { self.moduleName = name self.clangModulePath = modulePath self.clangModuleMapPath = moduleMapPath self.isFramework = false + self.isBridgingHeaderDependency = isBridgingHeaderDependency self.clangModuleCacheKey = moduleCacheKey } } -enum ModuleDependencyArtifactInfo: Codable { +@_spi(Testing) public enum ModuleDependencyArtifactInfo: Codable { case clang(ClangModuleArtifactInfo) case swift(SwiftModuleArtifactInfo) - func encode(to encoder: Encoder) throws { + @_spi(Testing) public func encode(to encoder: Encoder) throws { var container = encoder.singleValueContainer() switch self { case .swift(let swiftInfo): @@ -83,47 +87,15 @@ enum ModuleDependencyArtifactInfo: Codable { try container.encode(clangInfo) } } -} - -/// Describes a given module's batch dependency scanning input info -/// - Module Name -/// - Extra PCM build arguments (for Clang modules only) -/// - Dependency graph output path -public enum BatchScanModuleInfo: Encodable { - case swift(BatchScanSwiftModuleInfo) - case clang(BatchScanClangModuleInfo) -} -public struct BatchScanSwiftModuleInfo: Encodable { - var swiftModuleName: String - var output: String - - init(moduleName: String, outputPath: String) { - self.swiftModuleName = moduleName - self.output = outputPath - } -} - -public struct BatchScanClangModuleInfo: Encodable { - var clangModuleName: String - var arguments: String - var output: String - - init(moduleName: String, pcmArgs: String, outputPath: String) { - self.clangModuleName = moduleName - self.arguments = pcmArgs - self.output = outputPath - } -} - -public extension BatchScanModuleInfo { - func encode(to encoder: Encoder) throws { - var container = encoder.singleValueContainer() - switch self { - case .swift(let swiftInfo): - try container.encode(swiftInfo) - case .clang(let clangInfo): - try container.encode(clangInfo) + @_spi(Testing) public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + do { + let thing = try container.decode(SwiftModuleArtifactInfo.self) + self = .swift(thing) + } catch { + let thing = try container.decode(ClangModuleArtifactInfo.self) + self = .clang(thing) } } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift b/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift index 46937d619..a379c6abc 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift @@ -14,8 +14,6 @@ import class TSCBasic.DiagnosticsEngine import struct TSCBasic.AbsolutePath import struct TSCBasic.Diagnostic -@_implementationOnly import Yams - /// Holds the info about inputs needed to plan incremenal compilation /// A.k.a. BuildRecord was the legacy name public struct BuildRecord { @@ -40,155 +38,6 @@ public struct BuildRecord { self.buildEndTime = buildEndTime self.inputInfos = inputInfos } - - public enum SectionName: String, CaseIterable { - case swiftVersion = "version" - case argsHash = "options" - // Implement this for a smoother transition - case legacyBuildStartTime = "build_time" - case buildStartTime = "build_start_time" - case buildEndTime = "build_end_time" - case inputInfos = "inputs" - - var serializedName: String { rawValue } - } - - var allInputs: Set { - Set(inputInfos.map { $0.key }) - } -} - -// MARK: - Reading the old map and deciding whether to use it -public extension BuildRecord { - enum Error: Swift.Error { - case malformedYAML - case invalidKey - case missingTimeStamp - case missingInputSequenceNode - case missingInputEntryNode - case missingPriorBuildState - case unexpectedKey(String) - case malformed(SectionName) - - var reason: String { - switch self { - case .malformedYAML: - return "" - case .invalidKey: - return "" - case .missingTimeStamp: - return "could not read time value in build record" - case .missingInputSequenceNode: - return "no sequence node for input entry in build record" - case .missingInputEntryNode: - return "no input entry in build record" - case .missingPriorBuildState: - return "no previous build state in build record" - case .unexpectedKey(let key): - return "Unexpected key '\(key)'" - case .malformed(let section): - return "Malformed value for key '\(section.serializedName)'" - } - } - } - init(contents: String) throws { - guard let sections = try? Parser(yaml: contents, resolver: .basic, encoding: .utf8) - .singleRoot()?.mapping - else { - throw Error.malformedYAML - } - var argsHash: String? - var swiftVersion: String? - // Legacy driver does not disable incremental if no buildTime field. - var buildStartTime: TimePoint = .distantPast - var buildEndTime: TimePoint = .distantFuture - var inputInfos: [VirtualPath: InputInfo]? - for (key, value) in sections { - guard let k = key.string else { - throw Error.invalidKey - } - switch k { - case SectionName.swiftVersion.serializedName: - // There's a test that uses "" for an illegal value - guard let s = value.string, s != "" else { - break - } - swiftVersion = s - case SectionName.argsHash.serializedName: - guard let s = value.string, s != "" else { - break - } - argsHash = s - case SectionName.buildStartTime.serializedName, - SectionName.legacyBuildStartTime.serializedName: - buildStartTime = try Self.decodeDate(value, forInputInfo: false) - case SectionName.buildEndTime.serializedName: - buildEndTime = try Self.decodeDate(value, forInputInfo: false) - case SectionName.inputInfos.serializedName: - inputInfos = try Self.decodeInputInfos(value) - default: - throw Error.unexpectedKey(k) - } - } - // The legacy driver allows argHash to be absent to ease testing. - // Mimic the legacy driver for testing ease: If no `argsHash` section, - // record still matches. - guard let sv = swiftVersion else { - throw Error.malformed(.swiftVersion) - } - guard let iis = inputInfos else { - throw Error.malformed(.inputInfos) - } - guard let argsHash = argsHash else { - throw Error.malformed(.argsHash) - } - self.init(argsHash: argsHash, - swiftVersion: sv, - buildStartTime: buildStartTime, - buildEndTime: buildEndTime, - inputInfos: iis) - } - - private static func decodeDate( - _ node: Yams.Node, - forInputInfo: Bool - ) throws -> TimePoint { - guard let vals = node.sequence else { - if forInputInfo { - throw Error.missingInputSequenceNode - } else { - throw Error.missingTimeStamp - } - } - guard vals.count == 2, - let secs = vals[0].int, - let ns = vals[1].int - else { - throw Error.missingTimeStamp - } - return TimePoint(seconds: UInt64(secs), nanoseconds: UInt32(ns)) - } - - private static func decodeInputInfos( - _ node: Yams.Node - ) throws -> [VirtualPath: InputInfo] { - guard let map = node.mapping else { - throw BuildRecord.Error.malformed(.inputInfos) - } - var infos = [VirtualPath: InputInfo]() - for (keyNode, valueNode) in map { - guard let pathString = keyNode.string, - let path = try? VirtualPath(path: pathString) - else { - throw BuildRecord.Error.missingInputEntryNode - } - let previousModTime = try decodeDate(valueNode, forInputInfo: true) - let inputInfo = try InputInfo( - tag: valueNode.tag.description, previousModTime: previousModTime) - infos[path] = inputInfo - } - return infos - } } // MARK: - Creating and writing a new map @@ -222,82 +71,4 @@ extension BuildRecord { inputInfos: Dictionary(uniqueKeysWithValues: inputInfosArray) ) } - - /// Pass in `currentArgsHash` to ensure it is non-nil - public func encode(diagnosticEngine: DiagnosticsEngine) -> String? { - let pathsAndInfos = inputInfos.map { - input, inputInfo -> (String, InputInfo) in - return (input.name, inputInfo) - } - let inputInfosNode = Yams.Node( - pathsAndInfos - .sorted {$0.0 < $1.0} - .map {(Yams.Node($0.0, .implicit, .doubleQuoted), Self.encode($0.1))} - ) - let fieldNodes = [ - (SectionName.swiftVersion, Yams.Node(swiftVersion, .implicit, .doubleQuoted)), - (SectionName.argsHash, Yams.Node(argsHash, .implicit, .doubleQuoted)), - (SectionName.buildStartTime, Self.encode(buildStartTime)), - (SectionName.buildEndTime, Self.encode(buildEndTime)), - (SectionName.inputInfos, inputInfosNode ) - ] .map { (Yams.Node($0.0.serializedName), $0.1) } - - let buildRecordNode = Yams.Node(fieldNodes, .implicit, .block) - // let options = Yams.Emitter.Options(canonical: true) - do { - return try Yams.serialize(node: buildRecordNode, - width: -1, - sortKeys: false) - } catch { - diagnosticEngine.emit(.warning_could_not_serialize_build_record(error)) - return nil - } - } - - private static func encode(_ date: TimePoint, tag tagString: String? = nil) -> Yams.Node { - return Yams.Node( - [ Yams.Node(String(date.seconds)), Yams.Node(String(date.nanoseconds)) ], - tagString.map {Yams.Tag(Yams.Tag.Name(rawValue: $0))} ?? .implicit, - .flow) - } - - private static func encode(_ inputInfo: InputInfo) -> Yams.Node { - encode(inputInfo.previousModTime, tag: inputInfo.tag) - } - -} - -extension Diagnostic.Message { - static func warning_could_not_serialize_build_record(_ err: Error - ) -> Diagnostic.Message { - .warning("next compile won't be incremental; Could not serialize build record: \(err.localizedDescription)") - } - static func warning_could_not_write_build_record_not_absolutePath( - _ path: VirtualPath - ) -> Diagnostic.Message { - .warning("next compile won't be incremental; build record path was not absolute: \(path)") - } - static func warning_could_not_write_build_record(_ path: AbsolutePath - ) -> Diagnostic.Message { - .warning("next compile won't be incremental; could not write build record to \(path)") - } -} - - -// MARK: - reading -extension InputInfo { - fileprivate init( - tag: String, - previousModTime: TimePoint - ) throws { - guard let status = Status(identifier: tag) else { - throw BuildRecord.Error.missingPriorBuildState - } - self.init(status: status, previousModTime: previousModTime) - } -} - -// MARK: - writing -extension InputInfo { - fileprivate var tag: String { status.identifier } } diff --git a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift index f327faa78..2e531dc88 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift @@ -177,29 +177,6 @@ import class Dispatch.DispatchQueue try? fileSystem.removeFileTree(absPath) } - func removeInterModuleDependencyGraph() { - guard let absPath = interModuleDependencyGraphPath.absolutePath else { - return - } - try? fileSystem.removeFileTree(absPath) - } - - func readOutOfDateInterModuleDependencyGraph( - reporter: IncrementalCompilationState.Reporter? - ) -> InterModuleDependencyGraph? { - let decodedGraph: InterModuleDependencyGraph - do { - let contents = try fileSystem.readFileContents(interModuleDependencyGraphPath).cString - decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self, - from: Data(contents.utf8)) - } catch { - return nil - } - reporter?.report("Read inter-module dependency graph", interModuleDependencyGraphPath) - - return decodedGraph - } - func jobFinished(job: Job, result: ProcessResult) { self.confinementQueue.sync { finishedJobResults.append(JobResult(job, result)) @@ -220,11 +197,11 @@ import class Dispatch.DispatchQueue /// A build-record-relative path to the location of a serialized copy of the /// driver's inter-module dependency graph. - var interModuleDependencyGraphPath: VirtualPath { + var dependencyScanSerializedResultPath: VirtualPath { let filename = buildRecordPath.basenameWithoutExt return buildRecordPath .parentDirectory - .appending(component: filename + ".moduledeps") + .appending(component: filename + ".swiftmoduledeps") } /// Directory to emit dot files into diff --git a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift index 52ad83447..c404d90e6 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DependencyGraphDotFileWriter.swift @@ -10,7 +10,6 @@ // //===----------------------------------------------------------------------===// -import TSCBasic // <<< import protocol TSCBasic.WritableByteStream // MARK: - Asking to write dot files / interface @@ -126,7 +125,7 @@ extension ModuleDependencyGraph.Node: ExportableNode { extension ExportableNode { fileprivate func emit(id: Int, to out: inout WritableByteStream, _ t: InternedStringTable) { - out <<< DotFileNode(id: id, node: self, in: t).description <<< "\n" + out.send("\(DotFileNode(id: id, node: self, in: t).description)\n") } fileprivate func label(in t: InternedStringTable) -> String { @@ -225,11 +224,11 @@ fileprivate struct DOTDependencyGraphSerializer: Interne } private func emitPrelude() { - out <<< "digraph " <<< graphID.quoted <<< " {\n" + out.send("digraph \(graphID.quoted) {\n") } private mutating func emitLegend() { for dummy in DependencyKey.Designator.oneOfEachKind { - out <<< DotFileNode(forLegend: dummy).description <<< "\n" + out.send("\(DotFileNode(forLegend: dummy).description)\n") } } private mutating func emitNodes() { @@ -250,12 +249,12 @@ fileprivate struct DOTDependencyGraphSerializer: Interne private func emitArcs() { graph.forEachExportableArc { (def: Graph.Node, use: Graph.Node) in if include(def: def, use: use) { - out <<< DotFileArc(defID: nodeIDs[def]!, useID: nodeIDs[use]!).description <<< "\n" + out.send("\(DotFileArc(defID: nodeIDs[def]!, useID: nodeIDs[use]!).description)\n") } } } private func emitPostlude() { - out <<< "\n}\n" + out.send("\n}\n") } func include(_ n: Graph.Node) -> Bool { diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift index 22a17798b..770cde663 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift @@ -26,7 +26,6 @@ extension IncrementalCompilationState { let showJobLifecycle: Bool let alwaysRebuildDependents: Bool let interModuleDependencyGraph: InterModuleDependencyGraph? - let explicitModuleDependenciesGuaranteedUpToDate: Bool /// If non-null outputs information for `-driver-show-incremental` for input path private let reporter: Reporter? @@ -47,18 +46,18 @@ extension IncrementalCompilationState { self.alwaysRebuildDependents = initialState.incrementalOptions.contains( .alwaysRebuildDependents) self.interModuleDependencyGraph = interModuleDependencyGraph - self.explicitModuleDependenciesGuaranteedUpToDate = - initialState.upToDatePriorInterModuleDependencyGraph != nil ? true : false self.reporter = reporter } public func compute(batchJobFormer: inout Driver) throws -> FirstWave { return try blockingConcurrentAccessOrMutation { - let (initiallySkippedCompileGroups, mandatoryJobsInOrder) = + let (initiallySkippedCompileJobs, skippedNonCompileJobs, mandatoryJobsInOrder, afterCompiles) = try computeInputsAndGroups(batchJobFormer: &batchJobFormer) return FirstWave( - initiallySkippedCompileGroups: initiallySkippedCompileGroups, - mandatoryJobsInOrder: mandatoryJobsInOrder) + initiallySkippedCompileJobs: initiallySkippedCompileJobs, + skippedNonCompileJobs: skippedNonCompileJobs, + mandatoryJobsInOrder: mandatoryJobsInOrder, + jobsAfterCompiles: afterCompiles) } } } @@ -76,33 +75,43 @@ extension IncrementalCompilationState.FirstWaveComputer { /// At this stage the graph will have all external dependencies found in the swiftDeps or in the priors /// listed in fingerprintExternalDependencies. private func computeInputsAndGroups(batchJobFormer: inout Driver) - throws -> (initiallySkippedCompileGroups: [TypedVirtualPath: CompileJobGroup], - mandatoryJobsInOrder: [Job]) + throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job], + skippedNonCompileJobs: [Job], + mandatoryJobsInOrder: [Job], + jobsAfterCompiles: [Job]) { - let compileGroups = + let compileJobs = Dictionary(uniqueKeysWithValues: - jobsInPhases.compileGroups.map { ($0.primaryInput, $0) }) + jobsInPhases.compileJobs.map { ($0.primaryInputs[0], $0) }) let buildRecord = self.moduleDependencyGraph.buildRecord - guard !buildRecord.inputInfos.isEmpty else { - func everythingIsMandatory() - throws -> (initiallySkippedCompileGroups: [TypedVirtualPath: CompileJobGroup], - mandatoryJobsInOrder: [Job]) - { - let mandatoryCompileGroupsInOrder = self.inputFiles.swiftSourceFiles.compactMap { - input -> CompileJobGroup? in - compileGroups[input.typedFile] - } + let jobCreatingPch = jobsInPhases.beforeCompiles.first { $0.kind == .generatePCH } + + func everythingIsMandatory() + throws -> (initiallySkippedCompileJobs: [TypedVirtualPath: Job], + skippedNonCompileJobs: [Job], + mandatoryJobsInOrder: [Job], + jobsAfterCompiles: [Job]) + { + let mandatoryCompileJobsInOrder = self.inputFiles.swiftSourceFiles.compactMap { + input -> Job? in + compileJobs[input.typedFile] + } - let mandatoryJobsInOrder = try - jobsInPhases.beforeCompiles + - batchJobFormer.formBatchedJobs( - mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()}, - showJobLifecycle: showJobLifecycle) + let mandatoryJobsInOrder = try + jobsInPhases.beforeCompiles + + batchJobFormer.formBatchedJobs( + mandatoryCompileJobsInOrder, + showJobLifecycle: showJobLifecycle, + jobCreatingPch: jobCreatingPch) + + moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation) + return (initiallySkippedCompileJobs: [:], + skippedNonCompileJobs: [], + mandatoryJobsInOrder: mandatoryJobsInOrder, + jobsAfterCompiles: jobsInPhases.afterCompiles) + } - moduleDependencyGraph.setPhase(to: .buildingAfterEachCompilation) - return (initiallySkippedCompileGroups: [:], - mandatoryJobsInOrder: mandatoryJobsInOrder) - } + guard !buildRecord.inputInfos.isEmpty else { return try everythingIsMandatory() } moduleDependencyGraph.setPhase(to: .updatingAfterCompilation) @@ -112,142 +121,38 @@ extension IncrementalCompilationState.FirstWaveComputer { moduleDependencyGraph, buildRecord) - let initiallySkippedCompileGroups = compileGroups.filter { initiallySkippedInputs.contains($0.key) } + let initiallySkippedCompileJobs = compileJobs.filter { initiallySkippedInputs.contains($0.key) } - let mandatoryCompileGroupsInOrder = inputFiles.compactMap { - input -> CompileJobGroup? in - initiallySkippedInputs.contains(input) - ? nil - : compileGroups[input] + let mandatoryCompileJobsInOrder = inputFiles.compactMap { + input -> Job? in + return initiallySkippedInputs.contains(input) ? nil : compileJobs[input] } - let mandatoryBeforeCompilesJobs = try computeMandatoryBeforeCompilesJobs() let batchedCompilationJobs = try batchJobFormer.formBatchedJobs( - mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()}, - showJobLifecycle: showJobLifecycle) + mandatoryCompileJobsInOrder, + showJobLifecycle: showJobLifecycle, + jobCreatingPch: jobCreatingPch) // In the case where there are no compilation jobs to run on this build (no source-files were changed), // we can skip running `beforeCompiles` jobs if we also ensure that none of the `afterCompiles` jobs // have any dependencies on them. let skipAllJobs = batchedCompilationJobs.isEmpty ? !nonVerifyAfterCompileJobsDependOnBeforeCompileJobs() : false - let mandatoryJobsInOrder = skipAllJobs ? [] : mandatoryBeforeCompilesJobs + batchedCompilationJobs - return (initiallySkippedCompileGroups: initiallySkippedCompileGroups, - mandatoryJobsInOrder: mandatoryJobsInOrder) - } - - /// We must determine if any of the module dependencies require re-compilation - /// Since we know that a prior dependency graph was not completely up-to-date, - /// there must be at least *some* dependencies that require being re-built. - /// - /// If a dependency is deemed as requiring a re-build, then every module - /// between it and the root (source module being built by this driver - /// instance) must also be re-built. - private func computeInvalidatedModuleDependencies(on moduleDependencyGraph: InterModuleDependencyGraph) - throws -> Set { - let mainModuleInfo = moduleDependencyGraph.mainModule - var modulesRequiringRebuild: Set = [] - var validatedModules: Set = [] - // Scan from the main module's dependencies to avoid reporting - // the main module itself in the results. - for dependencyId in mainModuleInfo.directDependencies ?? [] { - try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, - pathSoFar: [], visitedValidated: &validatedModules, - modulesRequiringRebuild: &modulesRequiringRebuild) - } - - reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild)) - return modulesRequiringRebuild - } - - /// Perform a postorder DFS to locate modules which are out-of-date with respect - /// to their inputs. Upon encountering such a module, add it to the set of invalidated - /// modules, along with the path from the root to this module. - private func outOfDateModuleScan(on moduleDependencyGraph: InterModuleDependencyGraph, - from moduleId: ModuleDependencyId, - pathSoFar: [ModuleDependencyId], - visitedValidated: inout Set, - modulesRequiringRebuild: inout Set) throws { - let moduleInfo = try moduleDependencyGraph.moduleInfo(of: moduleId) - let isMainModule = moduleId == .swift(moduleDependencyGraph.mainModuleName) - - // Routine to invalidate the path from root to this node - let invalidatePath = { (modulesRequiringRebuild: inout Set) in - if let reporter { - for pathModuleId in pathSoFar { - if !modulesRequiringRebuild.contains(pathModuleId) && - !isMainModule { - let pathModuleInfo = try moduleDependencyGraph.moduleInfo(of: pathModuleId) - reporter.reportExplicitDependencyWillBeReBuilt(pathModuleId.moduleNameForDiagnostic, - reason: "Invalidated by downstream dependency") - } - } - } - modulesRequiringRebuild.formUnion(pathSoFar) - } + let beforeCompileJobs = skipAllJobs ? [] : jobsInPhases.beforeCompiles + var skippedNonCompileJobs = skipAllJobs ? jobsInPhases.beforeCompiles : [] - // Routine to invalidate this node and the path that led to it - let invalidateOutOfDate = { (modulesRequiringRebuild: inout Set) in - reporter?.reportExplicitDependencyWillBeReBuilt(moduleId.moduleNameForDiagnostic, reason: "Out-of-date") - modulesRequiringRebuild.insert(moduleId) - try invalidatePath(&modulesRequiringRebuild) - } - - // Visit the module's dependencies - for dependencyId in moduleInfo.directDependencies ?? [] { - if !visitedValidated.contains(dependencyId) { - let newPath = pathSoFar + [moduleId] - try outOfDateModuleScan(on: moduleDependencyGraph, from: dependencyId, pathSoFar: newPath, - visitedValidated: &visitedValidated, - modulesRequiringRebuild: &modulesRequiringRebuild) - } - } - - if modulesRequiringRebuild.contains(moduleId) { - try invalidatePath(&modulesRequiringRebuild) - } else if try !IncrementalCompilationState.IncrementalDependencyAndInputSetup.verifyModuleDependencyUpToDate(moduleID: moduleId, moduleInfo: moduleInfo, - fileSystem: fileSystem, reporter: reporter) { - try invalidateOutOfDate(&modulesRequiringRebuild) - } else { - // Only if this module is known to be up-to-date with respect to its inputs - // and dependencies do we mark it as visited. We may need to re-visit - // out-of-date modules in order to collect all possible paths to them. - visitedValidated.insert(moduleId) - } - } - - /// In an explicit module build, filter out dependency module pre-compilation tasks - /// for modules up-to-date from a prior compile. - private func computeMandatoryBeforeCompilesJobs() throws -> [Job] { - // In an implicit module build, we have nothing to filter/compute here - guard let moduleDependencyGraph = interModuleDependencyGraph else { - return jobsInPhases.beforeCompiles - } - - // If a prior compile's dependency graph was fully up-to-date, we can skip - // re-building all dependency modules. - guard !self.explicitModuleDependenciesGuaranteedUpToDate else { - return jobsInPhases.beforeCompiles.filter { $0.kind != .generatePCM && - $0.kind != .compileModuleFromInterface } - } - - // Determine which module pre-build jobs must be re-run - let modulesRequiringReBuild = - try computeInvalidatedModuleDependencies(on: moduleDependencyGraph) - - // Filter the `.generatePCM` and `.compileModuleFromInterface` jobs for - // modules which do *not* need re-building. - let mandatoryBeforeCompilesJobs = jobsInPhases.beforeCompiles.filter { job in - switch job.kind { - case .generatePCM: - return modulesRequiringReBuild.contains(.clang(job.moduleName)) - case .compileModuleFromInterface: - return modulesRequiringReBuild.contains(.swift(job.moduleName)) - default: - return true + // Schedule emitModule job together with verify module interface job. + let afterCompileJobs = jobsInPhases.afterCompiles.compactMap { job -> Job? in + if skipAllJobs && job.kind == .verifyModuleInterface { + skippedNonCompileJobs.append(job) + return nil } + return job } - - return mandatoryBeforeCompilesJobs + let mandatoryJobsInOrder = beforeCompileJobs + batchedCompilationJobs + return (initiallySkippedCompileJobs: initiallySkippedCompileJobs, + skippedNonCompileJobs: skippedNonCompileJobs, + mandatoryJobsInOrder: mandatoryJobsInOrder, + jobsAfterCompiles: afterCompileJobs) } /// Determine if any of the jobs in the `afterCompiles` group depend on outputs produced by jobs in @@ -255,9 +160,11 @@ extension IncrementalCompilationState.FirstWaveComputer { private func nonVerifyAfterCompileJobsDependOnBeforeCompileJobs() -> Bool { let beforeCompileJobOutputs = jobsInPhases.beforeCompiles.reduce(into: Set(), { (pathSet, job) in pathSet.formUnion(job.outputs) }) - let afterCompilesDependnigJobs = jobsInPhases.afterCompiles.filter {postCompileJob in postCompileJob.inputs.contains(where: beforeCompileJobOutputs.contains)} - if afterCompilesDependnigJobs.isEmpty || afterCompilesDependnigJobs.allSatisfy({ $0.kind == .verifyModuleInterface }) { + let afterCompilesDependingJobs = jobsInPhases.afterCompiles.filter {postCompileJob in postCompileJob.inputs.contains(where: beforeCompileJobOutputs.contains)} + if afterCompilesDependingJobs.isEmpty { return false + } else if afterCompilesDependingJobs.allSatisfy({ $0.kind == .verifyModuleInterface }) { + return jobsInPhases.beforeCompiles.contains { $0.kind == .generatePCM || $0.kind == .compileModuleFromInterface } } else { return true } @@ -269,7 +176,7 @@ extension IncrementalCompilationState.FirstWaveComputer { _ moduleDependencyGraph: ModuleDependencyGraph, _ buildRecord: BuildRecord ) -> Set { - let allGroups = jobsInPhases.compileGroups + let allCompileJobs = jobsInPhases.compileJobs // Input == source file let changedInputs = computeChangedInputs(moduleDependencyGraph, buildRecord) @@ -289,9 +196,9 @@ extension IncrementalCompilationState.FirstWaveComputer { reporter.report("Has malformed dependency source; will queue", input) } } - let inputsMissingOutputs = allGroups.compactMap { + let inputsMissingOutputs = allCompileJobs.compactMap { $0.outputs.contains { (try? !fileSystem.exists($0.file)) ?? true } - ? $0.primaryInput + ? $0.primaryInputs[0] : nil } if let reporter = reporter { @@ -370,8 +277,8 @@ extension IncrementalCompilationState.FirstWaveComputer { _ moduleDependencyGraph: ModuleDependencyGraph, _ outOfDateBuildRecord: BuildRecord ) -> [ChangedInput] { - jobsInPhases.compileGroups.compactMap { group in - let input = group.primaryInput + jobsInPhases.compileJobs.compactMap { job in + let input = job.primaryInputs[0] let modDate = buildRecordInfo.compilationInputModificationDates[input] ?? .distantFuture let inputInfo = outOfDateBuildRecord.inputInfos[input.file] let previousCompilationStatus = inputInfo?.status ?? .newlyAdded diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift index 1396cde37..90d482292 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift @@ -22,7 +22,7 @@ extension IncrementalCompilationState { /// /// This state is modified during the incremental build. All accesses must /// be protected by the confinement queue. - fileprivate var skippedCompileGroups: [TypedVirtualPath: CompileJobGroup] + fileprivate var skippedCompileJobs: [TypedVirtualPath: Job] /// Sadly, has to be `var` for formBatchedJobs /// @@ -34,14 +34,17 @@ extension IncrementalCompilationState { /// fileprivate in order to control concurrency. fileprivate let moduleDependencyGraph: ModuleDependencyGraph + fileprivate let jobCreatingPch: Job? fileprivate let reporter: Reporter? - init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup], + init(skippedCompileJobs: [TypedVirtualPath: Job], _ moduleDependencyGraph: ModuleDependencyGraph, + _ jobCreatingPch: Job?, _ driver: inout Driver) { - self.skippedCompileGroups = skippedCompileGroups + self.skippedCompileJobs = skippedCompileJobs self.moduleDependencyGraph = moduleDependencyGraph self.reporter = moduleDependencyGraph.info.reporter + self.jobCreatingPch = jobCreatingPch self.driver = driver } } @@ -61,7 +64,7 @@ extension IncrementalCompilationState.ProtectedState { mutationSafetyPrecondition() // batch in here to protect the Driver from concurrent access return try collectUnbatchedJobsDiscoveredToBeNeededAfterFinishing(job: finishedJob) - .map {try driver.formBatchedJobs($0, showJobLifecycle: driver.showJobLifecycle)} + .map {try driver.formBatchedJobs($0, showJobLifecycle: driver.showJobLifecycle, jobCreatingPch: jobCreatingPch)} } /// Remember a job (group) that is before a compile or a compile itself. @@ -109,7 +112,7 @@ extension IncrementalCompilationState.ProtectedState { } self.reporter?.report( "Failed to read some dependencies source; compiling everything", input) - return TransitivelyInvalidatedSwiftSourceFileSet(skippedCompileGroups.keys.swiftSourceFiles) + return TransitivelyInvalidatedSwiftSourceFileSet(skippedCompileJobs.keys.swiftSourceFiles) } /// Find the jobs that now must be run that were not originally known to be needed. @@ -117,17 +120,17 @@ extension IncrementalCompilationState.ProtectedState { for invalidatedInputs: Set ) throws -> [Job] { mutationSafetyPrecondition() - return invalidatedInputs.flatMap { input -> [Job] in - if let group = skippedCompileGroups.removeValue(forKey: input.typedFile) { - let primaryInputs = group.compileJob.primarySwiftSourceFiles + return invalidatedInputs.compactMap { input -> Job? in + if let job = skippedCompileJobs.removeValue(forKey: input.typedFile) { + let primaryInputs = job.primarySwiftSourceFiles assert(primaryInputs.count == 1) assert(primaryInputs[0] == input) self.reporter?.report("Scheduling invalidated", input) - return group.allJobs() + return job } else { self.reporter?.report("Tried to schedule invalidated input again", input) - return [] + return nil } } } @@ -138,13 +141,12 @@ extension IncrementalCompilationState.ProtectedState { extension IncrementalCompilationState.ProtectedState { var skippedCompilationInputs: Set { accessSafetyPrecondition() - return Set(skippedCompileGroups.keys) + return Set(skippedCompileJobs.keys) } public var skippedJobs: [Job] { accessSafetyPrecondition() - return skippedCompileGroups.values - .sorted {$0.primaryInput.file.name < $1.primaryInput.file.name} - .flatMap {$0.allJobs()} + return skippedCompileJobs.values + .sorted {$0.primaryInputs[0].file.name < $1.primaryInputs[0].file.name} } func writeGraph(to path: VirtualPath, diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift index a8731eda9..8ee2b86c6 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift @@ -50,8 +50,6 @@ extension IncrementalCompilationState { let graph: ModuleDependencyGraph /// Information about the last known compilation, incl. the location of build artifacts such as the dependency graph. let buildRecordInfo: BuildRecordInfo - /// Record about the compiled module's explicit module dependencies from a prior compile. - let upToDatePriorInterModuleDependencyGraph: InterModuleDependencyGraph? /// A set of inputs invalidated by external changes. let inputsInvalidatedByExternals: TransitivelyInvalidatedSwiftSourceFileSet /// Compiler options related to incremental builds. @@ -66,11 +64,16 @@ extension IncrementalCompilationState { /// The set of compile jobs we can definitely skip given the state of the /// incremental dependency graph and the status of the input files for this /// incremental build. - let initiallySkippedCompileGroups: [TypedVirtualPath: CompileJobGroup] + let initiallySkippedCompileJobs: [TypedVirtualPath: Job] + /// The non-compile jobs that can be skipped given the state of the + /// incremental build. + let skippedNonCompileJobs: [Job] /// All of the pre-compile or compilation job (groups) known to be required /// for the first wave to execute. /// The primaries could be other than .swift files, i.e. .sib let mandatoryJobsInOrder: [Job] + /// The job after compilation that needs to run. + let jobsAfterCompiles: [Job] } } @@ -90,12 +93,6 @@ extension Driver { because: "it is not compatible with \(compilerMode)")) return false } - guard !parsedOptions.hasArgument(.embedBitcode) else { - diagnosticEngine.emit( - .remark_incremental_compilation_has_been_disabled( - because: "is not currently compatible with embedding LLVM IR bitcode")) - return false - } return true } } @@ -293,11 +290,6 @@ extension IncrementalCompilationState { report("\(message): \(externalDependency.shortDescription)") } - // Emits a remark indicating a need for a dependency scanning invocation - func reportExplicitBuildMustReScan(_ why: String) { - report("Incremental build must re-run dependency scan: \(why)") - } - func reportExplicitDependencyOutOfDate(_ moduleName: String, inputPath: String) { report("Dependency module \(moduleName) is older than input file \(inputPath)") @@ -308,10 +300,19 @@ extension IncrementalCompilationState { report("Dependency module '\(moduleOutputPath)' will be re-built: \(reason)") } + func reportPriorExplicitDependencyStale(_ moduleOutputPath: String, + reason: String) { + report("Dependency module '\(moduleOutputPath)' info is stale: \(reason)") + } + func reportExplicitDependencyReBuildSet(_ modules: [ModuleDependencyId]) { report("Following explicit module dependencies will be re-built: [\(modules.map { $0.moduleNameForDiagnostic }.sorted().joined(separator: ", "))]") } + func reportExplicitDependencyMissingFromCAS(_ moduleName: String) { + report("Dependency module \(moduleName) is missing from CAS") + } + // Emits a remark indicating incremental compilation has been disabled. func reportDisablingIncrementalBuild(_ why: String) { report("Disabling incremental build: \(why)") @@ -369,16 +370,6 @@ extension IncrementalCompilationState { /// integrity of the driver's dependency graph and aborts if any errors /// are detected. public static let verifyDependencyGraphAfterEveryImport = Options(rawValue: 1 << 3) - /// Enables the cross-module incremental build infrastructure. - /// - /// FIXME: This option is transitory. We intend to make this the - /// default behavior. This option should flip to a "disable" bit after that. - public static let enableCrossModuleIncrementalBuild = Options(rawValue: 1 << 4) - /// Enables an optimized form of start-up for the incremental build state - /// that reads the dependency graph from a serialized format on disk instead - /// of reading O(N) swiftdeps files. - public static let readPriorsFromModuleDependencyGraph = Options(rawValue: 1 << 5) - /// Enables additional handling of explicit module build artifacts: /// Additional reading and writing of the inter-module dependency graph. public static let explicitModuleBuild = Options(rawValue: 1 << 6) @@ -405,7 +396,6 @@ extension IncrementalCompilationState { to path: VirtualPath, _ buildRecord: BuildRecord ) throws { - precondition(info.isCrossModuleIncrementalBuildEnabled) try blockingConcurrentAccessOrMutationToProtectedState { try $0.writeGraph( to: path, @@ -421,55 +411,6 @@ extension IncrementalCompilationState { } } -extension IncrementalCompilationState { - enum WriteInterModuleDependencyGraphError: LocalizedError { - case noDependencyGraph - var errorDescription: String? { - switch self { - case .noDependencyGraph: - return "No inter-module dependency graph present" - } - } - } - - func writeInterModuleDependencyGraph(_ buildRecordInfo: BuildRecordInfo?) throws { - // If the explicit module build is not happening, there will not be a graph to write - guard info.explicitModuleBuild else { - return - } - guard let recordInfo = buildRecordInfo else { - throw WriteDependencyGraphError.noBuildRecordInfo - } - guard let interModuleDependencyGraph = self.upToDateInterModuleDependencyGraph else { - throw WriteInterModuleDependencyGraphError.noDependencyGraph - } - do { - let encoder = JSONEncoder() -#if os(Linux) || os(Android) - encoder.outputFormatting = [.prettyPrinted] -#else - if #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) { - encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes] - } -#endif - let data = try encoder.encode(interModuleDependencyGraph) - try fileSystem.writeFileContents(recordInfo.interModuleDependencyGraphPath, - bytes: ByteString(data), - atomically: true) - } catch { - throw IncrementalCompilationState.WriteDependencyGraphError.couldNotWrite( - path: recordInfo.interModuleDependencyGraphPath, error: error) - } - - } - - @_spi(Testing) public static func removeInterModuleDependencyGraphFile(_ driver: Driver) { - if let path = driver.buildRecordInfo?.interModuleDependencyGraphPath { - try? driver.fileSystem.removeFileTree(path) - } - } -} - // MARK: - OutputFileMap extension OutputFileMap { func onlySourceFilesHaveSwiftDeps() -> Bool { diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift index 709562bb9..25b08290a 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift @@ -44,6 +44,9 @@ public final class IncrementalCompilationState { /// Jobs to run *after* the last compile, for instance, link-editing. public let jobsAfterCompiles: [Job] + /// The skipped non compile jobs. + public let skippedJobsNonCompile: [Job] + public let info: IncrementalCompilationState.IncrementalDependencyAndInputSetup internal let upToDateInterModuleDependencyGraph: InterModuleDependencyGraph? @@ -54,32 +57,31 @@ public final class IncrementalCompilationState { driver: inout Driver, jobsInPhases: JobsInPhases, initialState: InitialStateForPlanning, - interModuleDependencyGraph: InterModuleDependencyGraph? + interModuleDepGraph: InterModuleDependencyGraph? ) throws { let reporter = initialState.incrementalOptions.contains(.showIncremental) ? Reporter(diagnosticEngine: driver.diagnosticEngine, outputFileMap: driver.outputFileMap) : nil - reporter?.reportOnIncrementalImports( - initialState.incrementalOptions.contains(.enableCrossModuleIncrementalBuild)) - let firstWave = try FirstWaveComputer( initialState: initialState, jobsInPhases: jobsInPhases, driver: driver, - interModuleDependencyGraph: interModuleDependencyGraph, + interModuleDependencyGraph: interModuleDepGraph, reporter: reporter) .compute(batchJobFormer: &driver) self.info = initialState.graph.info - self.upToDateInterModuleDependencyGraph = interModuleDependencyGraph + self.upToDateInterModuleDependencyGraph = interModuleDepGraph self.protectedState = ProtectedState( - skippedCompileGroups: firstWave.initiallySkippedCompileGroups, + skippedCompileJobs: firstWave.initiallySkippedCompileJobs, initialState.graph, + jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH}), &driver) self.mandatoryJobsInOrder = firstWave.mandatoryJobsInOrder - self.jobsAfterCompiles = jobsInPhases.afterCompiles + self.jobsAfterCompiles = firstWave.jobsAfterCompiles + self.skippedJobsNonCompile = firstWave.skippedNonCompileJobs } /// Allow concurrent access to while preventing mutation of ``IncrementalCompilationState/protectedState`` @@ -104,10 +106,3 @@ extension IncrementalCompilationState: IncrementalCompilationSynchronizer { info.incrementalCompilationQueue } } - -fileprivate extension IncrementalCompilationState.Reporter { - func reportOnIncrementalImports(_ enabled: Bool) { - report( - "\(enabled ? "Enabling" : "Disabling") incremental cross-module building") - } -} diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift index e09fd069b..2ee0498ff 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift @@ -58,9 +58,6 @@ extension IncrementalCompilationState { ).computeInitialStateForPlanning(driver: &driver) else { Self.removeDependencyGraphFile(driver) - if options.contains(.explicitModuleBuild) { - Self.removeInterModuleDependencyGraphFile(driver) - } return nil } @@ -84,11 +81,10 @@ extension IncrementalCompilationState { if driver.parsedOptions.contains(veriOpt) { options.formUnion(.verifyDependencyGraphAfterEveryImport) } - if driver.parsedOptions.hasFlag(positive: .enableIncrementalImports, - negative: .disableIncrementalImports, - default: true) { - options.formUnion(.enableCrossModuleIncrementalBuild) - options.formUnion(.readPriorsFromModuleDependencyGraph) + if !driver.parsedOptions.hasFlag(positive: .enableIncrementalImports, + negative: .disableIncrementalImports, + default: true) { + driver.diagnosticEngine.emit(.warning("ignoring obseleted option -disable-incremental-imports")) } if driver.parsedOptions.contains(.driverExplicitModuleBuild) { options.formUnion(.explicitModuleBuild) @@ -97,134 +93,6 @@ extension IncrementalCompilationState { } } -/// Validate if a prior inter-module dependency graph is still valid -extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { - static func readAndValidatePriorInterModuleDependencyGraph( - driver: inout Driver, - buildRecordInfo: BuildRecordInfo, - reporter: IncrementalCompilationState.Reporter? - ) throws -> InterModuleDependencyGraph? { - // Attempt to read a serialized inter-module dependency graph from a prior build - guard let priorInterModuleDependencyGraph = - buildRecordInfo.readOutOfDateInterModuleDependencyGraph(reporter: reporter), - let priorImports = priorInterModuleDependencyGraph.mainModule.directDependencies?.map({ $0.moduleName }) else { - reporter?.reportExplicitBuildMustReScan("Could not read inter-module dependency graph at \(buildRecordInfo.interModuleDependencyGraphPath)") - return nil - } - - // Verify that import sets match - let currentImports = try driver.performImportPrescan().imports - guard Set(priorImports) == Set(currentImports) else { - reporter?.reportExplicitBuildMustReScan("Target import set has changed.") - return nil - } - - // Verify that each dependnecy is up-to-date with respect to its inputs - guard try verifyInterModuleDependenciesUpToDate(in: priorInterModuleDependencyGraph, - buildRecordInfo: buildRecordInfo, - reporter: reporter) else { - reporter?.reportExplicitBuildMustReScan("Not all dependencies are up-to-date.") - return nil - } - - reporter?.report("Confirmed prior inter-module dependency graph is up-to-date at: \(buildRecordInfo.interModuleDependencyGraphPath)") - return priorInterModuleDependencyGraph - } - - static func verifyModuleDependencyUpToDate(moduleID: ModuleDependencyId, moduleInfo: ModuleInfo, - fileSystem: FileSystem, - reporter: IncrementalCompilationState.Reporter?) throws -> Bool { - // Verify that the specified input exists and is older than the specified output - let verifyInputOlderThanOutputModTime: (String, VirtualPath, VirtualPath, TimePoint) -> Bool = - { moduleName, inputPath, outputPath, outputModTime in - guard let inputModTime = - try? fileSystem.lastModificationTime(for: inputPath) else { - reporter?.report("Unable to 'stat' \(inputPath.description)") - return false - } - if inputModTime > outputModTime { - reporter?.reportExplicitDependencyOutOfDate(moduleName, - inputPath: inputPath.description) - return false - } - return true - } - - switch moduleInfo.details { - case .swift(let swiftDetails): - guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)) else { - reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") - return false - } - if let moduleInterfacePath = swiftDetails.moduleInterfacePath { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(moduleInterfacePath.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - if let bridgingHeaderPath = swiftDetails.bridgingHeaderPath { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(bridgingHeaderPath.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - for bridgingSourceFile in swiftDetails.bridgingSourceFiles ?? [] { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - VirtualPath.lookup(bridgingSourceFile.path), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - case .clang(_): - guard let outputModTime = try? fileSystem.lastModificationTime(for: VirtualPath.lookup(moduleInfo.modulePath.path)) else { - reporter?.report("Module output not found: '\(moduleID.moduleNameForDiagnostic)'") - return false - } - for inputSourceFile in moduleInfo.sourceFiles ?? [] { - if !verifyInputOlderThanOutputModTime(moduleID.moduleName, - try VirtualPath(path: inputSourceFile), - VirtualPath.lookup(moduleInfo.modulePath.path), - outputModTime) { - return false - } - } - case .swiftPrebuiltExternal(_): - // TODO: We have to give-up here until we have a way to verify the timestamp of the binary module. - // We can do better here by knowing if this module hasn't change - which would allows us to not - // invalidate any of the dependencies that depend on it. - reporter?.report("Unable to verify binary module dependency up-to-date: \(moduleID.moduleNameForDiagnostic)") - return false; - case .swiftPlaceholder(_): - // TODO: This should never ever happen. Hard error? - return false; - } - return true - } - - /// For each direct and transitive module dependency, check if any of the inputs are newer than the output - static func verifyInterModuleDependenciesUpToDate(in graph: InterModuleDependencyGraph, - buildRecordInfo: BuildRecordInfo, - reporter: IncrementalCompilationState.Reporter?) throws -> Bool { - for module in graph.modules { - if module.key.moduleName == graph.mainModuleName { - continue - } - if try !verifyModuleDependencyUpToDate(moduleID: module.key, - moduleInfo: module.value, - fileSystem: buildRecordInfo.fileSystem, - reporter: reporter) { - return false - } - } - return true - } -} - /// Builds the `InitialState` /// Also bundles up an bunch of configuration info extension IncrementalCompilationState { @@ -248,18 +116,12 @@ extension IncrementalCompilationState { @_spi(Testing) public let dependencyDotFilesIncludeExternals: Bool = true @_spi(Testing) public let dependencyDotFilesIncludeAPINotes: Bool = false - @_spi(Testing) public var readPriorsFromModuleDependencyGraph: Bool { - options.contains(.readPriorsFromModuleDependencyGraph) - } @_spi(Testing) public var explicitModuleBuild: Bool { options.contains(.explicitModuleBuild) } @_spi(Testing) public var alwaysRebuildDependents: Bool { options.contains(.alwaysRebuildDependents) } - @_spi(Testing) public var isCrossModuleIncrementalBuildEnabled: Bool { - options.contains(.enableCrossModuleIncrementalBuild) - } @_spi(Testing) public var verifyDependencyGraphAfterEveryImport: Bool { options.contains(.verifyDependencyGraphAfterEveryImport) } @@ -295,23 +157,8 @@ extension IncrementalCompilationState { return nil } - // If a valid build record could not be produced, do not bother here - let priorInterModuleDependencyGraph: InterModuleDependencyGraph? - if options.contains(.explicitModuleBuild) { - if priors.graph.buildRecord.inputInfos.isEmpty { - reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.") - priorInterModuleDependencyGraph = nil - } else { - priorInterModuleDependencyGraph = try Self.readAndValidatePriorInterModuleDependencyGraph( - driver: &driver, buildRecordInfo: buildRecordInfo, reporter: reporter) - } - } else { - priorInterModuleDependencyGraph = nil - } - return InitialStateForPlanning( graph: priors.graph, buildRecordInfo: buildRecordInfo, - upToDatePriorInterModuleDependencyGraph: priorInterModuleDependencyGraph, inputsInvalidatedByExternals: priors.fileSet, incrementalOptions: options) } @@ -341,12 +188,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { /// caller must ensure scheduling of those private func computeGraphAndInputsInvalidatedByExternals() -> PriorState? { return blockingConcurrentAccessOrMutation { - if readPriorsFromModuleDependencyGraph { - return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() - } - // Every external is added, but don't want to compile an unchanged input that has an import - // so just changed, not changedOrAdded. - return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() + return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() } } @@ -406,54 +248,6 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup { return PriorState(graph: graph, fileSet: inputsInvalidatedByExternals) } - /// Builds a graph - /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file. - /// Does not cope with disappeared inputs - /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph: - /// caller must ensure scheduling of those - /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise - /// does nothing special. - private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() -> PriorState? { - guard - let contents = try? fileSystem.readFileContents(self.buildRecordInfo.buildRecordPath).cString - else { - reporter?.report("Incremental compilation could not read build record at ", self.buildRecordInfo.buildRecordPath) - reporter?.reportDisablingIncrementalBuild("could not read build record") - return nil - } - - func failedToReadOutOfDateMap(_ reason: String) { - let why = "malformed build record file\(reason.isEmpty ? "" : (" " + reason))" - reporter?.report( - "Incremental compilation has been disabled due to \(why)", self.buildRecordInfo.buildRecordPath) - reporter?.reportDisablingIncrementalBuild(why) - } - - do { - guard let buildRecord = try self.validateBuildRecord(BuildRecord(contents: contents)) else { - return nil - } - - let graph = ModuleDependencyGraph.createForBuildingFromSwiftDeps(buildRecord, self) - var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedSwiftSourceFileSet() - for input in self.inputFiles { - guard let invalidatedInputs = - graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: SwiftSourceFile(input.fileHandle)) - else { - return nil - } - inputsInvalidatedByChangedExternals.formUnion(invalidatedInputs) - } - reporter?.report("Created dependency graph from swiftdeps files") - return PriorState(graph: graph, fileSet: inputsInvalidatedByChangedExternals) - } catch let error as BuildRecord.Error { - failedToReadOutOfDateMap(error.reason) - return nil - } catch { - return nil - } - } - private func buildEmptyGraphAndCompileEverything() -> PriorState { let buildRecord = BuildRecord( argsHash: self.buildRecordInfo.currentArgsHash, diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index 04a0b848b..621d9778a 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -414,10 +414,6 @@ extension ModuleDependencyGraph { func findNodesInvalidated( by integrand: ExternalIntegrand ) -> DirectlyInvalidatedNodeSet { - guard self.info.isCrossModuleIncrementalBuildEnabled else { - return indiscriminatelyFindNodesInvalidated(by: integrand, - .incrementalImportsIsDisabled) - } // If the integrand has no fingerprint, it's academic, cannot integrate it incrementally. guard integrand.externalDependency.fingerprint != nil else { return indiscriminatelyFindNodesInvalidated(by: integrand, @@ -438,7 +434,6 @@ extension ModuleDependencyGraph { func incrementallyFindNodesInvalidated( by integrand: ExternalIntegrand ) -> DirectlyInvalidatedNodeSet { - assert(self.info.isCrossModuleIncrementalBuildEnabled) accessSafetyPrecondition() // Better not be reading swiftdeps one-by-one for a selective compilation precondition(self.phase != .buildingFromSwiftDeps) diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift index 06f3ec1f6..2cec31e3d 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift @@ -38,6 +38,12 @@ extension ModuleDependencyGraph { /*@_spi(Testing)*/ public var key: DependencyKey { keyAndFingerprint.key } /*@_spi(Testing)*/ public var fingerprint: InternedString? { keyAndFingerprint.fingerprint } + /// When integrating a change, the driver finds untraced nodes so it can kick off jobs that have not been + /// kicked off yet. (Within any one driver invocation, compiling a source file is idempotent.) + /// When reading a serialized, prior graph, *don't* recover this state, since it will be a new driver + /// invocation that has not kicked off any compiles yet. + @_spi(Testing) public private(set) var isTraced: Bool = false + /// Each Node corresponds to a declaration, somewhere. If the definition has been already found, /// the `definitionLocation` will point to it. /// If uses are encountered before the definition (in reading swiftdeps files), the `definitionLocation` @@ -46,11 +52,6 @@ extension ModuleDependencyGraph { /// compilation. @_spi(Testing) public let definitionLocation: DefinitionLocation - /// When integrating a change, the driver finds untraced nodes so it can kick off jobs that have not been - /// kicked off yet. (Within any one driver invocation, compiling a source file is idempotent.) - /// When reading a serialized, prior graph, *don't* recover this state, since it will be a new driver - /// invocation that has not kicked off any compiles yet. - @_spi(Testing) public private(set) var isTraced: Bool = false private let cachedHash: Int diff --git a/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift b/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift index 3d8a60f1c..ad2c46f1d 100644 --- a/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift +++ b/Sources/SwiftDriver/Jobs/APIDigesterJobs.swift @@ -46,7 +46,10 @@ extension Driver { var commandLine = [Job.ArgTemplate]() commandLine.appendFlag("-dump-sdk") - try addCommonDigesterOptions(&commandLine, modulePath: modulePath, mode: mode) + try addCommonDigesterOptions(&commandLine, + modulePath: modulePath, + swiftModuleInterfacePath: self.moduleOutputPaths.swiftInterfacePath, + mode: mode) commandLine.appendFlag(.o) commandLine.appendPath(VirtualPath.lookup(outputPath)) @@ -69,12 +72,16 @@ extension Driver { case .api: return nil case .abi: - return abiDescriptorPath + return moduleOutputPaths.abiDescriptorFilePath } } guard let currentABI = getDescriptorPath(for: mode) else { // we don't have existing descriptor to use so we have to load the module from interface/swiftmodule - return try digesterCompareToBaselineJob(modulePath: modulePath, baselinePath: baselinePath, mode: digesterMode) + return try digesterCompareToBaselineJob( + modulePath: modulePath, + swiftModuleInterfacePath: self.moduleOutputPaths.swiftInterfacePath, + baselinePath: baselinePath, + mode: digesterMode) } var commandLine = [Job.ArgTemplate]() commandLine.appendFlag("-diagnose-sdk") @@ -105,14 +112,20 @@ extension Driver { ) } - mutating func digesterCompareToBaselineJob(modulePath: VirtualPath.Handle, baselinePath: VirtualPath.Handle, mode: DigesterMode) throws -> Job { + mutating func digesterCompareToBaselineJob(modulePath: VirtualPath.Handle, + swiftModuleInterfacePath: VirtualPath.Handle?, + baselinePath: VirtualPath.Handle, + mode: DigesterMode) throws -> Job { var commandLine = [Job.ArgTemplate]() commandLine.appendFlag("-diagnose-sdk") commandLine.appendFlag("-disable-fail-on-error") commandLine.appendFlag("-baseline-path") commandLine.appendPath(VirtualPath.lookup(baselinePath)) - try addCommonDigesterOptions(&commandLine, modulePath: modulePath, mode: mode) + try addCommonDigesterOptions(&commandLine, + modulePath: modulePath, + swiftModuleInterfacePath: swiftModuleInterfacePath, + mode: mode) var serializedDiagnosticsPath: VirtualPath.Handle? if let arg = parsedOptions.getLastArgument(.serializeBreakingChangesPath)?.asSingle { @@ -130,7 +143,7 @@ extension Driver { var inputs: [TypedVirtualPath] = [.init(file: modulePath, type: .swiftModule), .init(file: baselinePath, type: mode.baselineFileType)] // If a module interface was emitted, treat it as an input in ABI mode. - if let interfacePath = self.swiftInterfacePath, mode == .abi { + if let interfacePath = swiftModuleInterfacePath, mode == .abi { inputs.append(.init(file: interfacePath, type: .swiftInterface)) } @@ -147,6 +160,7 @@ extension Driver { private mutating func addCommonDigesterOptions(_ commandLine: inout [Job.ArgTemplate], modulePath: VirtualPath.Handle, + swiftModuleInterfacePath: VirtualPath.Handle?, mode: DigesterMode) throws { commandLine.appendFlag("-module") commandLine.appendFlag(moduleOutputInfo.name) @@ -160,7 +174,7 @@ extension Driver { let searchPath = VirtualPath.lookup(modulePath).parentDirectory commandLine.appendFlag(.I) commandLine.appendPath(searchPath) - if let interfacePath = self.swiftInterfacePath { + if let interfacePath = swiftModuleInterfacePath { let interfaceSearchPath = VirtualPath.lookup(interfacePath).parentDirectory if interfaceSearchPath != searchPath { commandLine.appendFlag(.I) @@ -180,6 +194,10 @@ extension Driver { commandLine.appendPath(VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path)) try commandLine.appendAll(.I, from: &parsedOptions) + for systemImport in parsedOptions.arguments(for: .Isystem) { + commandLine.appendFlag(.isystem) + commandLine.appendFlag(systemImport.argument.asSingle) + } try commandLine.appendAll(.F, from: &parsedOptions) for systemFramework in parsedOptions.arguments(for: .Fsystem) { commandLine.appendFlag(.iframework) diff --git a/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift b/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift index d1cd15bc3..c7ed8558a 100644 --- a/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift +++ b/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift @@ -18,7 +18,18 @@ import struct TSCBasic.RelativePath // FIXME: Also handle Cygwin and MinGW extension Driver { /*@_spi(Testing)*/ public var isAutolinkExtractJobNeeded: Bool { - [.elf, .wasm].contains(targetTriple.objectFormat) && lto == nil + mutating get { + switch targetTriple.objectFormat { + case .wasm where !parsedOptions.isEmbeddedEnabled: + fallthrough + + case .elf: + return lto == nil && linkerOutputType != nil + + default: + return false + } + } } mutating func autolinkExtractJob(inputs: [TypedVirtualPath]) throws -> Job? { @@ -33,8 +44,8 @@ extension Driver { let dir = firstInput.file.parentDirectory // Go through a bit of extra rigmarole to keep the "./" out of the name for // the sake of the tests. - let output: VirtualPath = dir == .temporary(RelativePath(".")) - ? VirtualPath.createUniqueTemporaryFile(RelativePath(outputBasename)) + let output: VirtualPath = dir == .temporary(try RelativePath(validating: ".")) + ? try VirtualPath.createUniqueTemporaryFile(RelativePath(validating: outputBasename)) : dir.appending(component: outputBasename) commandLine.append(contentsOf: inputs.map { .path($0.file) }) diff --git a/Sources/SwiftDriver/Jobs/BackendJob.swift b/Sources/SwiftDriver/Jobs/BackendJob.swift deleted file mode 100644 index d160fc954..000000000 --- a/Sources/SwiftDriver/Jobs/BackendJob.swift +++ /dev/null @@ -1,81 +0,0 @@ -//===--------------- BackendJob.swift - Swift Backend Job -------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - - -extension Driver { - /// Form a backend job. - mutating func backendJob(input: TypedVirtualPath, - baseInput: TypedVirtualPath?, - addJobOutputs: ([TypedVirtualPath]) -> Void) - throws -> Job { - var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - var inputs = [TypedVirtualPath]() - var outputs = [TypedVirtualPath]() - - commandLine.appendFlag("-frontend") - addCompileModeOption(outputType: compilerOutputType, commandLine: &commandLine) - - // Add input arguments. - commandLine.appendFlag(.primaryFile) - commandLine.appendPath(input.file) - inputs.append(input) - - commandLine.appendFlag(.embedBitcode) - - // -embed-bitcode only supports a restricted set of flags. - commandLine.appendFlag(.target) - commandLine.appendFlag(targetTriple.triple) - - // Enable address top-byte ignored in the ARM64 backend. - if targetTriple.arch == .aarch64 { - commandLine.appendFlag(.Xllvm) - commandLine.appendFlag("-aarch64-use-tbi") - } - - // Handle the CPU and its preferences. - try commandLine.appendLast(.targetCpu, from: &parsedOptions) - - // Enable optimizations, but disable all LLVM-IR-level transformations. - try commandLine.appendLast(in: .O, from: &parsedOptions) - commandLine.appendFlag(.disableLlvmOptzns) - - try commandLine.appendLast(.parseStdlib, from: &parsedOptions) - - commandLine.appendFlag(.moduleName) - commandLine.appendFlag(moduleOutputInfo.name) - - // Add the output file argument if necessary. - if let compilerOutputType = compilerOutputType { - // If there is no baseInput (singleCompileMode), primary output computation - // is not input-specific, therefore it does not matter which input is passed. - let output = try computePrimaryOutput(for: baseInput ?? input, - outputType: compilerOutputType, - isTopLevel: isTopLevelOutput(type: compilerOutputType)) - commandLine.appendFlag(.o) - commandLine.appendPath(output.file) - outputs.append(output) - } - - addJobOutputs(outputs) - - return Job( - moduleName: moduleOutputInfo.name, - kind: .backend, - tool: try toolchain.resolvedTool(.swiftCompiler), - commandLine: commandLine, - displayInputs: inputs, - inputs: inputs, - primaryInputs: [], - outputs: outputs - ) - } -} diff --git a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift index 73a457c2b..821191251 100644 --- a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift +++ b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift @@ -56,7 +56,7 @@ extension Array where Element == Job.ArgTemplate { /// Append an option's spelling to the command line arguments. mutating func appendFlag(_ option: Option) { switch option.kind { - case .flag, .joinedOrSeparate, .remaining, .separate: + case .flag, .joinedOrSeparate, .remaining, .separate, .multiArg: break case .commaJoined, .input, .joined: fatalError("Option cannot be appended as a flag: \(option)") @@ -95,7 +95,7 @@ extension Array where Element == Job.ArgTemplate { assert(!option.attributes.contains(.argumentIsPath)) appendFlag(option.spelling + argument.asMultiple.joined(separator: ",")) - case .remaining: + case .remaining, .multiArg: appendFlag(option.spelling) for arg in argument.asMultiple { try appendSingleArgument(option: option, argument: arg) diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index dd9a7e378..1ede3f8f2 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -66,17 +66,17 @@ extension Driver { } if !isTopLevel { - return TypedVirtualPath(file: VirtualPath.createUniqueTemporaryFile(.init(baseName.appendingFileTypeExtension(outputType))).intern(), + return TypedVirtualPath(file: try VirtualPath.createUniqueTemporaryFile(.init(validating: baseName.appendingFileTypeExtension(outputType))).intern(), type: outputType) } - return TypedVirtualPath(file: try useWorkingDirectory(.init(baseName.appendingFileTypeExtension(outputType))).intern(), type: outputType) + return TypedVirtualPath(file: try useWorkingDirectory(try .init(validating: baseName.appendingFileTypeExtension(outputType))).intern(), type: outputType) } /// Is this compile job top-level func isTopLevelOutput(type: FileType?) -> Bool { switch type { - case .assembly, .sil, .raw_sil, .llvmIR, .ast, .jsonDependencies, .sib, .raw_sib, - .importedModules, .indexData: + case .assembly, .sil, .raw_sil, .raw_llvmIr, .llvmIR, .ast, .jsonDependencies, .sib, + .raw_sib, .importedModules, .indexData: return true case .object: return (linkerOutputType == nil) @@ -92,12 +92,13 @@ extension Driver { case .swiftModule: return compilerMode.isSingleCompilation && moduleOutputInfo.output?.isTopLevel ?? false case .swift, .image, .dSYM, .dependencies, .emitModuleDependencies, .autolink, - .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, + .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .diagnostics, .emitModuleDiagnostics, .objcHeader, .swiftDeps, .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline, - .swiftConstValues, nil: + .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, + .cachedDiagnostics, .jsonSupportedFeatures, nil: return false } } @@ -112,21 +113,26 @@ extension Driver { outputType: FileType?, commandLine: inout [Job.ArgTemplate]) throws -> ([TypedVirtualPath], [TypedVirtualPath]) { - let useInputFileList: Bool - if let allSourcesFileList = allSourcesFileList { - useInputFileList = true + let useInputFileList = shouldUseInputFileList + if let sourcesFileList = allSourcesFileList { commandLine.appendFlag(.filelist) - commandLine.appendPath(allSourcesFileList) - } else { - useInputFileList = false + commandLine.appendPath(sourcesFileList) + } else if shouldUseInputFileList { + let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation) + let remappedSourcesFileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "sources"), + .list(swiftInputs.map{ return remapPath($0.file) })) + // Remember the filelist created. + self.allSourcesFileList = remappedSourcesFileList + commandLine.appendFlag(.filelist) + commandLine.appendPath(remappedSourcesFileList) } let usePrimaryInputFileList = primaryInputs.count > fileListThreshold if usePrimaryInputFileList { // primary file list commandLine.appendFlag(.primaryFilelist) - let fileList = VirtualPath.createUniqueFilelist(RelativePath("primaryInputs"), - .list(primaryInputs.map(\.file))) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "primaryInputs"), + .list(primaryInputs.map{ return remapPath($0.file) })) commandLine.appendPath(fileList) } @@ -166,12 +172,11 @@ extension Driver { let isPrimary = usesPrimaryFileInputs && primaryInputFiles.contains(input) if isPrimary { if !usePrimaryInputFileList { - commandLine.appendFlag(.primaryFile) - commandLine.appendPath(input.file) + try addPathOption(option: .primaryFile, path: input.file, to:&commandLine) } } else { if !useInputFileList { - commandLine.appendPath(input.file) + try addPathArgument(input.file, to: &commandLine) } } @@ -226,7 +231,8 @@ extension Driver { outputType: FileType?, addJobOutputs: ([TypedVirtualPath]) -> Void, pchCompileJob: Job?, - emitModuleTrace: Bool) + emitModuleTrace: Bool, + produceCacheKey: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] @@ -272,6 +278,8 @@ extension Driver { primaryInputs: primaryInputs, inputsGeneratingCodeCount: inputsGeneratingCodeCount, inputOutputMap: &inputOutputMap, + moduleOutputInfo: self.moduleOutputInfo, + moduleOutputPaths: self.moduleOutputPaths, includeModuleTracePath: emitModuleTrace, indexFilePath: indexFilePath) @@ -285,13 +293,13 @@ extension Driver { } try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .compile) - - // FIXME: MSVC runtime flags + try addRuntimeLibraryFlags(commandLine: &commandLine) if Driver.canDoCrossModuleOptimization(parsedOptions: &parsedOptions) && // For historical reasons, -cross-module-optimization turns on "aggressive" CMO // which is different from "default" CMO. - !parsedOptions.hasArgument(.CrossModuleOptimization) { + !parsedOptions.hasArgument(.CrossModuleOptimization) && + !parsedOptions.hasArgument(.EnableCMOEverything) { assert(!emitModuleSeparately, "Cannot emit module separately with cross-module-optimization") commandLine.appendFlag("-enable-default-cmo") } @@ -311,8 +319,8 @@ extension Driver { // Add primary outputs. if primaryOutputs.count > fileListThreshold { commandLine.appendFlag(.outputFilelist) - let fileList = VirtualPath.createUniqueFilelist(RelativePath("outputs"), - .list(primaryOutputs.map { $0.file })) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "outputs"), + .list(primaryOutputs.map { $0.file })) commandLine.appendPath(fileList) } else { for primaryOutput in primaryOutputs { @@ -325,8 +333,8 @@ extension Driver { if !primaryIndexUnitOutputs.isEmpty { if primaryIndexUnitOutputs.count > fileListThreshold { commandLine.appendFlag(.indexUnitOutputPathFilelist) - let fileList = VirtualPath.createUniqueFilelist(RelativePath("index-unit-outputs"), - .list(primaryIndexUnitOutputs.map { $0.file })) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "index-unit-outputs"), + .list(primaryIndexUnitOutputs.map { $0.file })) commandLine.appendPath(fileList) } else { for primaryIndexUnitOutput in primaryIndexUnitOutputs { @@ -365,11 +373,14 @@ extension Driver { try commandLine.appendLast(.trackSystemDependencies, from: &parsedOptions) try commandLine.appendLast(.CrossModuleOptimization, from: &parsedOptions) + try commandLine.appendLast(.EnableCMOEverything, from: &parsedOptions) try commandLine.appendLast(.ExperimentalPerformanceAnnotations, from: &parsedOptions) - try commandLine.appendLast(.disableAutolinkingRuntimeCompatibility, from: &parsedOptions) + try commandLine.appendLast(.runtimeCompatibilityVersion, from: &parsedOptions) + try commandLine.appendLast(.disableAutolinkingRuntimeCompatibility, from: &parsedOptions) try commandLine.appendLast(.disableAutolinkingRuntimeCompatibilityDynamicReplacements, from: &parsedOptions) try commandLine.appendLast(.disableAutolinkingRuntimeCompatibilityConcurrency, from: &parsedOptions) + try commandLine.appendLast(.checkApiAvailabilityOnly, from: &parsedOptions) try addCommonSymbolGraphOptions(commandLine: &commandLine, @@ -390,6 +401,24 @@ extension Driver { } else { displayInputs = primaryInputs } + // The cache key for compilation is created one per input file, and each cache key contains all the output + // files for that specific input file. All the module level output files are attached to the cache key for + // the first input file. Only the input files that produce the output will have a cache key. This behavior + // needs to match the cache key creation logic in swift-frontend. + let cacheContributingInputs = inputs.enumerated().reduce(into: [(TypedVirtualPath, Int)]()) { result, input in + guard input.element.type == .swift else { return } + let singleInputKey = TypedVirtualPath(file: OutputFileMap.singleInputKey, type: .swift) + if inputOutputMap[singleInputKey] != nil { + // If singleInputKey exists, that means only the first swift file produces outputs. + if result.isEmpty { + result.append((input.element, input.offset)) + } + } else if !inputOutputMap[input.element, default: []].isEmpty { + // Otherwise, add all the inputs that produce output. + result.append((input.element, input.offset)) + } + } + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: cacheContributingInputs) return Job( moduleName: moduleOutputInfo.name, @@ -400,6 +429,7 @@ extension Driver { inputs: inputs, primaryInputs: primaryInputs, outputs: outputs, + outputCacheKeys: cacheKeys, inputOutputMap: inputOutputMap ) } @@ -435,6 +465,8 @@ extension FileType { return .emitSibgen case .sib: return .emitSib + case .raw_llvmIr: + return .emitIrgen case .llvmIR: return .emitIr case .llvmBitcode: @@ -455,14 +487,17 @@ extension FileType { return .printTargetInfo case .jsonCompilerFeatures: return .emitSupportedFeatures + case .jsonSupportedFeatures: + return .printSupportedFeatures case .swift, .dSYM, .autolink, .dependencies, .emitModuleDependencies, .swiftDocumentation, .pcm, .diagnostics, .emitModuleDiagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, - .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, + .bitstreamOptimizationRecord, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, .jsonABIBaseline, - .swiftConstValues: + .swiftConstValues, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, + .cachedDiagnostics: fatalError("Output type can never be a primary output") } } diff --git a/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift index 7068d6e00..27f4c6157 100644 --- a/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift @@ -68,10 +68,10 @@ extension DarwinToolchain { // Same as an executable, but with the -dylib flag linkerTool = .dynamicLinker commandLine.appendFlag("-dynamiclib") - addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, - commandLine: &commandLine, - inputs: inputs, - linkerOutputType: linkerOutputType) + try addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) try addDynamicLinkerFlags(targetInfo: targetInfo, parsedOptions: &parsedOptions, commandLine: &commandLine, @@ -81,10 +81,10 @@ extension DarwinToolchain { case .executable: linkerTool = .dynamicLinker - addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, - commandLine: &commandLine, - inputs: inputs, - linkerOutputType: linkerOutputType) + try addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) try addDynamicLinkerFlags(targetInfo: targetInfo, parsedOptions: &parsedOptions, commandLine: &commandLine, @@ -95,10 +95,10 @@ extension DarwinToolchain { case .staticLibrary: linkerTool = .staticLinker(lto) commandLine.appendFlag(.static) - addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, - commandLine: &commandLine, - inputs: inputs, - linkerOutputType: linkerOutputType) + try addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) } // Add the output @@ -111,7 +111,7 @@ extension DarwinToolchain { private func addLinkInputs(shouldUseInputFileList: Bool, commandLine: inout [Job.ArgTemplate], inputs: [TypedVirtualPath], - linkerOutputType: LinkOutputType) { + linkerOutputType: LinkOutputType) throws { // inputs LinkFileList if shouldUseInputFileList { commandLine.appendFlag(.filelist) @@ -119,7 +119,6 @@ extension DarwinToolchain { var inputModules = [VirtualPath]() for input in inputs { if input.type == .swiftModule && linkerOutputType != .staticLibrary { - inputPaths.append(input.file) inputModules.append(input.file) } else if input.type == .object { inputPaths.append(input.file) @@ -129,8 +128,8 @@ extension DarwinToolchain { inputPaths.append(input.file) } } - let fileList = VirtualPath.createUniqueFilelist(RelativePath("inputs.LinkFileList"), - .list(inputPaths)) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "inputs.LinkFileList"), + .list(inputPaths)) commandLine.appendPath(fileList) if linkerOutputType != .staticLibrary { for module in inputModules { @@ -177,11 +176,13 @@ extension DarwinToolchain { } } - if let arg = parsedOptions.getLastArgument(.useLd) { - commandLine.appendFlag("-fuse-ld=\(arg.asSingle)") + if let arg = parsedOptions.getLastArgument(.useLd)?.asSingle { + commandLine.appendFlag("-fuse-ld=\(arg)") } - try commandLine.appendLast(.ldPath, from: &parsedOptions) + if let arg = parsedOptions.getLastArgument(.ldPath)?.asSingle { + commandLine.append(.joinedOptionAndPath("--ld-path=", try VirtualPath(path: arg))) + } let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) for opt in fSystemArgs { @@ -207,11 +208,13 @@ extension DarwinToolchain { .sorted() // Sort so we get a stable, testable order .joined(separator: ",") commandLine.appendFlag("-fsanitize=\(sanitizerNames)") + + if parsedOptions.contains(.sanitizeStableAbiEQ) { + commandLine.appendFlag("-fsanitize-stable-abi") + } } - if parsedOptions.contains(.embedBitcode) { - commandLine.appendFlag("-fembed-bitcode") - } else if parsedOptions.contains(.embedBitcodeMarker) { + if parsedOptions.contains(.embedBitcodeMarker) { commandLine.appendFlag("-fembed-bitcode=marker") } @@ -232,13 +235,14 @@ extension DarwinToolchain { commandLine.appendFlag("--target=\(targetTriple.triple)") if let variantTriple = targetInfo.targetVariant?.triple { assert(targetTriple.isValidForZipperingWithTriple(variantTriple)) - commandLine.appendFlag("-darwin-target-variant=\(variantTriple.triple)") + commandLine.appendFlag("-darwin-target-variant") + commandLine.appendFlag(variantTriple.triple) } // On Darwin, we only support libc++. var cxxCompatEnabled = parsedOptions.hasArgument(.enableExperimentalCxxInterop) if let cxxInteropMode = parsedOptions.getLastArgument(.cxxInteroperabilityMode) { - if cxxInteropMode.asSingle == "swift-5.9" { + if cxxInteropMode.asSingle != "off" { cxxCompatEnabled = true } } @@ -266,13 +270,7 @@ extension DarwinToolchain { from: &parsedOptions ) addLinkedLibArgs(to: &commandLine, parsedOptions: &parsedOptions) - // Because we invoke `clang` as the linker executable, we must still - // use `-Xlinker` for linker-specific arguments. - for linkerOpt in parsedOptions.arguments(for: .Xlinker) { - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag(linkerOpt.argument.asSingle) - } - try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + try addExtraClangLinkerArgs(to: &commandLine, parsedOptions: &parsedOptions) } } @@ -293,6 +291,10 @@ private extension DarwinPlatform { return ["watchos"] case .watchOS(.simulator): return ["watchossim", "watchos"] + case .visionOS(.device): + return ["xros"] + case .visionOS(.simulator): + return ["xrossim", "xros"] } } } diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index 468764d2f..0a8fde6db 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -17,6 +17,7 @@ extension Driver { mutating func addCommonModuleOptions( commandLine: inout [Job.ArgTemplate], outputs: inout [TypedVirtualPath], + moduleOutputPaths: SupplementalModuleTargetOutputPaths, isMergeModule: Bool ) throws { // Add supplemental outputs. @@ -28,12 +29,16 @@ extension Driver { outputs.append(.init(file: path, type: type)) } - addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation) - addSupplementalOutput(path: moduleSourceInfoPath, flag: "-emit-module-source-info-path", type: .swiftSourceInfoFile) - addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface) - addSupplementalOutput(path: swiftPrivateInterfacePath, flag: "-emit-private-module-interface-path", type: .privateSwiftInterface) + addSupplementalOutput(path: moduleOutputPaths.moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation) + addSupplementalOutput(path: moduleOutputPaths.moduleSourceInfoPath, flag: "-emit-module-source-info-path", type: .swiftSourceInfoFile) + addSupplementalOutput(path: moduleOutputPaths.swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface) + addSupplementalOutput(path: moduleOutputPaths.swiftPrivateInterfacePath, flag: "-emit-private-module-interface-path", type: .privateSwiftInterface) + if let pkgName = packageName, !pkgName.isEmpty { + addSupplementalOutput(path: moduleOutputPaths.swiftPackageInterfacePath, flag: "-emit-package-module-interface-path", type: .packageSwiftInterface) + } addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader) addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd) + addSupplementalOutput(path: moduleOutputPaths.apiDescriptorFilePath, flag: "-emit-api-descriptor-path", type: .jsonAPIDescriptor) if isMergeModule { return @@ -74,8 +79,13 @@ extension Driver { } /// Form a job that emits a single module - @_spi(Testing) public mutating func emitModuleJob(pchCompileJob: Job?) throws -> Job { - let moduleOutputPath = moduleOutputInfo.output!.outputPath + @_spi(Testing) public mutating func emitModuleJob(pchCompileJob: Job?, isVariantJob: Bool = false) throws -> Job { + precondition(!isVariantJob || (isVariantJob && + variantModuleOutputInfo != nil && variantModuleOutputPaths != nil), + "target variant module requested without a target variant") + + let moduleOutputPath = isVariantJob ? variantModuleOutputInfo!.output!.outputPath : moduleOutputInfo.output!.outputPath + let moduleOutputPaths = isVariantJob ? variantModuleOutputPaths! : moduleOutputPaths var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] var outputs: [TypedVirtualPath] = [ @@ -86,7 +96,7 @@ extension Driver { // Add the inputs. for input in self.inputFiles where input.type.isPartOfSwiftCompilation { - commandLine.append(.path(input.file)) + try addPathArgument(input.file, to: &commandLine) inputs.append(input) } @@ -98,13 +108,15 @@ extension Driver { try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .emitModule) // FIXME: Add MSVC runtime library flags - try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: false) + try addCommonModuleOptions( + commandLine: &commandLine, + outputs: &outputs, + moduleOutputPaths: moduleOutputPaths, + isMergeModule: false) + try addCommonSymbolGraphOptions(commandLine: &commandLine) try commandLine.appendLast(.checkApiAvailabilityOnly, from: &parsedOptions) - if isFrontendArgSupported(.experimentalLazyTypecheck) { - try commandLine.appendLast(.experimentalLazyTypecheck, from: &parsedOptions) - } if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) { commandLine.appendFlag(.parseAsLibrary) @@ -113,11 +125,17 @@ extension Driver { let outputPath = VirtualPath.lookup(moduleOutputPath) commandLine.appendFlag(.o) commandLine.appendPath(outputPath) - if let abiPath = abiDescriptorPath { + if let abiPath = moduleOutputPaths.abiDescriptorFilePath { commandLine.appendFlag(.emitAbiDescriptorPath) commandLine.appendPath(abiPath.file) outputs.append(abiPath) } + let cacheContributingInputs = inputs.enumerated().reduce(into: [(TypedVirtualPath, Int)]()) { result, input in + // only the first swift input contributes cache key to an emit module job. + guard result.isEmpty, input.element.type == .swift else { return } + result.append((input.element, input.offset)) + } + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: cacheContributingInputs) return Job( moduleName: moduleOutputInfo.name, kind: .emitModule, @@ -125,7 +143,8 @@ extension Driver { commandLine: commandLine, inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys ) } @@ -135,7 +154,7 @@ extension Driver { moduleOutputInfo: ModuleOutputInfo, inputFiles: [TypedVirtualPath]) -> Bool { if moduleOutputInfo.output == nil || - !inputFiles.allSatisfy({ $0.type.isPartOfSwiftCompilation }) { + !inputFiles.contains(where: { $0.type.isPartOfSwiftCompilation }) { return false } diff --git a/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift b/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift index 4e02b7567..06fce203d 100644 --- a/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift @@ -37,8 +37,8 @@ extension Toolchain { // at least one so we fake it. // FIXME: Teach -emit-supported-features to not expect any inputs, like -print-target-info does. let dummyInputPath = - VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("dummyInput.swift"), - "".data(using: .utf8)!) + try VirtualPath.createUniqueTemporaryFileWithKnownContents(.init(validating: "dummyInput.swift"), + "".data(using: .utf8)!) commandLine.appendPath(dummyInputPath) inputs.append(TypedVirtualPath(file: dummyInputPath.intern(), type: .swift)) @@ -59,18 +59,19 @@ extension Toolchain { extension Driver { static func computeSupportedCompilerArgs(of toolchain: Toolchain, + libSwiftScan: SwiftScan?, parsedOptions: inout ParsedOptions, diagnosticsEngine: DiagnosticsEngine, fileSystem: FileSystem, executor: DriverExecutor) throws -> Set { - do { - if let supportedArgs = - try querySupportedCompilerArgsInProcess(of: toolchain, fileSystem: fileSystem) { - return supportedArgs + if let libSwiftScanInstance = libSwiftScan, + libSwiftScanInstance.canQuerySupportedArguments() { + do { + return try libSwiftScanInstance.querySupportedArguments() + } catch { + diagnosticsEngine.emit(.remark_inprocess_supported_features_query_failed(error.localizedDescription)) } - } catch { - diagnosticsEngine.emit(.warning_inprocess_supported_features_query_failed(error.localizedDescription)) } // Fallback: Invoke `swift-frontend -emit-supported-features` and decode the output @@ -88,20 +89,6 @@ extension Driver { return Set(decodedSupportedFlagList) } - static func querySupportedCompilerArgsInProcess(of toolchain: Toolchain, - fileSystem: FileSystem) - throws -> Set? { - let optionalSwiftScanLibPath = try toolchain.lookupSwiftScanLib() - if let swiftScanLibPath = optionalSwiftScanLibPath, - fileSystem.exists(swiftScanLibPath) { - let libSwiftScanInstance = try SwiftScan(dylib: swiftScanLibPath) - if libSwiftScanInstance.canQuerySupportedArguments() { - return try libSwiftScanInstance.querySupportedArguments() - } - } - return nil - } - static func computeSupportedCompilerFeatures(of toolchain: Toolchain, env: [String: String]) throws -> Set { struct FeatureInfo: Codable { diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 519029afa..c406f5241 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -51,6 +51,15 @@ extension Driver { /// If the driver is in Explicit Module Build mode, the dependency graph has been computed case computed } + + /// If the given option is specified but the frontend doesn't support it, throw an error. + func verifyFrontendSupportsOptionIfNecessary(_ option: Option) throws { + if parsedOptions.hasArgument(option) && !isFrontendArgSupported(option) { + diagnosticEngine.emit(.error_unsupported_opt_for_frontend(option: option)) + throw ErrorDiagnostics.emitted + } + } + /// Add frontend options that are common to different frontend invocations. mutating func addCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], @@ -75,25 +84,42 @@ extension Driver { break } + let isPlanJobForExplicitModule = parsedOptions.contains(.driverExplicitModuleBuild) && + moduleDependencyGraphUse == .computed + let jobNeedPathRemap: Bool // If in ExplicitModuleBuild mode and the dependency graph has been computed, add module // dependencies. // May also be used for generation of the dependency graph itself in ExplicitModuleBuild mode. - if (parsedOptions.contains(.driverExplicitModuleBuild) && - moduleDependencyGraphUse == .computed) { + if isPlanJobForExplicitModule { switch kind { case .generatePCH: try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) + jobNeedPathRemap = true case .compile, .emitModule, .interpret, .verifyModuleInterface: try addExplicitModuleBuildArguments(inputs: &inputs, commandLine: &commandLine) + jobNeedPathRemap = true case .backend, .mergeModule, .compileModuleFromInterface, .generatePCM, .dumpPCM, .repl, .printTargetInfo, .versionRequest, .autolinkExtract, .generateDSYM, .help, .link, .verifyDebugInfo, .scanDependencies, .emitSupportedFeatures, .moduleWrap, .generateAPIBaseline, .generateABIBaseline, .compareAPIBaseline, - .compareABIBaseline: - break // Do not support creating from dependency scanner output. + .compareABIBaseline, .printSupportedFeatures: + jobNeedPathRemap = false } + } else { + jobNeedPathRemap = false + } + + // Check if dependency scanner has put the job into direct clang cc1 mode. + // If dependency scanner put us into direct cc1 mode, avoid adding `-Xcc` options, since + // dependency scanner already adds needed flags and -Xcc options known by swift-driver are + // clang driver flags but not it requires cc1 flags. + let directModuleCC1Mode = commandLine.contains(Job.ArgTemplate.flag("-direct-clang-cc1-module-build")) + func appendXccFlag(_ flag: String) { + guard !directModuleCC1Mode else { return } + commandLine.appendFlag(.Xcc) + commandLine.appendFlag(flag) } if let variant = parsedOptions.getLastArgument(.targetVariant)?.asSingle { @@ -107,8 +133,10 @@ extension Driver { commandLine.appendFlag("-aarch64-use-tbi") } + let isEmbeddedEnabled = parsedOptions.isEmbeddedEnabled + // Enable or disable ObjC interop appropriately for the platform - if targetTriple.isDarwin { + if targetTriple.isDarwin && !isEmbeddedEnabled { commandLine.appendFlag(.enableObjcInterop) } else { commandLine.appendFlag(.disableObjcInterop) @@ -117,67 +145,91 @@ extension Driver { // Add flags for C++ interop try commandLine.appendLast(.enableExperimentalCxxInterop, from: &parsedOptions) try commandLine.appendLast(.cxxInteroperabilityMode, from: &parsedOptions) - if let stdlibVariant = parsedOptions.getLastArgument(.experimentalCxxStdlib)?.asSingle { - commandLine.appendFlag("-Xcc") - commandLine.appendFlag("-stdlib=\(stdlibVariant)") + + if isEmbeddedEnabled && parsedOptions.hasArgument(.enableLibraryEvolution) { + diagnosticEngine.emit(.error_no_library_evolution_embedded) + throw ErrorDiagnostics.emitted + } + + // Building embedded Swift requires WMO, unless we're not generating SIL. This allows modes like -index-file to work the same way they do when not using embedded Swift + if isEmbeddedEnabled && compilerOutputType?.requiresSILGen == true && + (!parsedOptions.hasArgument(.wmo) || !parsedOptions.hasArgument(.wholeModuleOptimization)) { + diagnosticEngine.emit(.error_need_wmo_embedded) + throw ErrorDiagnostics.emitted } + if isEmbeddedEnabled && parsedOptions.hasArgument(.enableObjcInterop) { + diagnosticEngine.emit(.error_no_objc_interop_embedded) + throw ErrorDiagnostics.emitted + } + + try verifyFrontendSupportsOptionIfNecessary(.disableUpcomingFeature) + try verifyFrontendSupportsOptionIfNecessary(.disableExperimentalFeature) + // Handle the CPU and its preferences. try commandLine.appendLast(.targetCpu, from: &parsedOptions) if let sdkPath = frontendTargetInfo.sdkPath?.path { - commandLine.appendFlag(.sdk) - commandLine.append(.path(VirtualPath.lookup(sdkPath))) + try addPathOption(option: .sdk, path: VirtualPath.lookup(sdkPath), to: &commandLine, remap: jobNeedPathRemap) } for args: (Option, Option) in [ (.visualcToolsRoot, .visualcToolsVersion), (.windowsSdkRoot, .windowsSdkVersion) ] { - let (rootArg, versionArg) = args - if let value = parsedOptions.getLastArgument(rootArg)?.asSingle, - isFrontendArgSupported(rootArg) { - commandLine.appendFlag(rootArg.spelling) - commandLine.appendPath(.init(value)) + let (rootOpt, versionOpt) = args + if let rootArg = parsedOptions.last(for: rootOpt), + isFrontendArgSupported(rootOpt) { + try addPathOption(rootArg, to: &commandLine, remap: jobNeedPathRemap) } - if let value = parsedOptions.getLastArgument(versionArg)?.asSingle, - isFrontendArgSupported(versionArg) { - commandLine.appendFlags(versionArg.spelling, value) + if let value = parsedOptions.getLastArgument(versionOpt)?.asSingle, + isFrontendArgSupported(versionOpt) { + commandLine.appendFlags(versionOpt.spelling, value) } } - try commandLine.appendAll(.I, from: &parsedOptions) - try commandLine.appendAll(.F, .Fsystem, from: &parsedOptions) - try commandLine.appendAll(.vfsoverlay, from: &parsedOptions) + // TODO: Can we drop all search paths for compile jobs for explicit module build? + try addAllArgumentsWithPath(.I, .Isystem, to: &commandLine, remap: jobNeedPathRemap) + try addAllArgumentsWithPath(.F, .Fsystem, to: &commandLine, remap: jobNeedPathRemap) + try addAllArgumentsWithPath(.vfsoverlay, to: &commandLine, remap: jobNeedPathRemap) if let gccToolchain = parsedOptions.getLastArgument(.gccToolchain) { - commandLine.appendFlag(.Xcc) - commandLine.appendFlag("--gcc-toolchain=\(gccToolchain.asSingle)") + appendXccFlag("--gcc-toolchain=\(gccToolchain.asSingle)") } try commandLine.appendLast(.AssertConfig, from: &parsedOptions) try commandLine.appendLast(.autolinkForceLoad, from: &parsedOptions) - if let colorOption = parsedOptions.last(for: .colorDiagnostics, .noColorDiagnostics) { - commandLine.appendFlag(colorOption.option) - } else if shouldColorDiagnostics() { + if parsedOptions.hasFlag(positive: .colorDiagnostics, negative: .noColorDiagnostics, default: shouldColorDiagnostics()) { commandLine.appendFlag(.colorDiagnostics) + appendXccFlag("-fcolor-diagnostics") + } else { + commandLine.appendFlag(.noColorDiagnostics) + appendXccFlag("-fno-color-diagnostics") } try commandLine.appendLast(.fixitAll, from: &parsedOptions) try commandLine.appendLast(.warnSwift3ObjcInferenceMinimal, .warnSwift3ObjcInferenceComplete, from: &parsedOptions) try commandLine.appendLast(.warnImplicitOverrides, from: &parsedOptions) + try commandLine.appendLast(.warnSoftDeprecated, from: &parsedOptions) try commandLine.appendLast(.typoCorrectionLimit, from: &parsedOptions) try commandLine.appendLast(.enableAppExtension, from: &parsedOptions) try commandLine.appendLast(.enableLibraryEvolution, from: &parsedOptions) try commandLine.appendLast(.enableTesting, from: &parsedOptions) try commandLine.appendLast(.enablePrivateImports, from: &parsedOptions) try commandLine.appendLast(in: .g, from: &parsedOptions) - try commandLine.appendLast(.debugInfoFormat, from: &parsedOptions) + if debugInfo.level != nil { + commandLine.appendFlag("-debug-info-format=\(debugInfo.format.rawValue)") + if isFrontendArgSupported(.dwarfVersion) { + commandLine.appendFlag("-dwarf-version=\(debugInfo.dwarfVersion)") + } + } try commandLine.appendLast(.importUnderlyingModule, from: &parsedOptions) try commandLine.appendLast(.moduleCachePath, from: &parsedOptions) try commandLine.appendLast(.moduleLinkName, from: &parsedOptions) + try commandLine.appendLast(.moduleAbiName, from: &parsedOptions) try commandLine.appendLast(.nostdimport, from: &parsedOptions) + try commandLine.appendLast(.nostdlibimport, from: &parsedOptions) try commandLine.appendLast(.parseStdlib, from: &parsedOptions) try commandLine.appendLast(.solverMemoryThreshold, from: &parsedOptions) try commandLine.appendLast(.valueRecursionThreshold, from: &parsedOptions) @@ -189,10 +241,24 @@ extension Driver { try commandLine.appendLast(.profileGenerate, from: &parsedOptions) try commandLine.appendLast(.profileUse, from: &parsedOptions) try commandLine.appendLast(.profileCoverageMapping, from: &parsedOptions) - try commandLine.appendLast(.warningsAsErrors, .noWarningsAsErrors, from: &parsedOptions) + try commandLine.appendLast(.debugInfoForProfiling, from: &parsedOptions) + if parsedOptions.hasArgument(.profileSampleUse) { + try commandLine.appendLast(.profileSampleUse, from: &parsedOptions) + // Use LLVM's "profi" to infer missing sample data from the profile. + commandLine.appendFlag(.Xllvm) + commandLine.appendFlag("-sample-profile-use-profi") + } + try commandLine.appendAllExcept( + includeList: [.warningTreating], + excludeList: [], + from: &parsedOptions + ) try commandLine.appendLast(.sanitizeEQ, from: &parsedOptions) try commandLine.appendLast(.sanitizeRecoverEQ, from: &parsedOptions) try commandLine.appendLast(.sanitizeAddressUseOdrIndicator, from: &parsedOptions) + if isFrontendArgSupported(.sanitizeStableAbiEQ) { + try commandLine.appendLast(.sanitizeStableAbiEQ, from: &parsedOptions) + } try commandLine.appendLast(.sanitizeCoverageEQ, from: &parsedOptions) try commandLine.appendLast(.static, from: &parsedOptions) try commandLine.appendLast(.swiftVersion, from: &parsedOptions) @@ -208,8 +274,10 @@ extension Driver { try commandLine.appendLast(.packageDescriptionVersion, from: &parsedOptions) try commandLine.appendLast(.serializeDiagnosticsPath, from: &parsedOptions) try commandLine.appendLast(.debugDiagnosticNames, from: &parsedOptions) + try commandLine.appendLast(.printDiagnosticGroups, from: &parsedOptions) try commandLine.appendLast(.scanDependencies, from: &parsedOptions) try commandLine.appendLast(.enableExperimentalConcisePoundFile, from: &parsedOptions) + try commandLine.appendLast(.experimentalPackageInterfaceLoad, from: &parsedOptions) try commandLine.appendLast(.printEducationalNotes, from: &parsedOptions) try commandLine.appendLast(.diagnosticStyle, from: &parsedOptions) try commandLine.appendLast(.locale, from: &parsedOptions) @@ -220,18 +288,25 @@ extension Driver { try commandLine.appendLast(.lto, from: &parsedOptions) try commandLine.appendLast(.accessNotesPath, from: &parsedOptions) try commandLine.appendLast(.enableActorDataRaceChecks, .disableActorDataRaceChecks, from: &parsedOptions) + if isFrontendArgSupported(.dumpAstFormat) { + try commandLine.appendLast(.dumpAstFormat, from: &parsedOptions) + } try commandLine.appendAll(.D, from: &parsedOptions) try commandLine.appendAll(.debugPrefixMap, .coveragePrefixMap, .filePrefixMap, from: &parsedOptions) try commandLine.appendAllArguments(.Xfrontend, from: &parsedOptions) try commandLine.appendLast(.warnConcurrency, from: &parsedOptions) - if isFrontendArgSupported(.enableExperimentalFeature) { - try commandLine.appendAll( - .enableExperimentalFeature, from: &parsedOptions) + if isFrontendArgSupported(.noAllocations) { + try commandLine.appendLast(.noAllocations, from: &parsedOptions) } - if isFrontendArgSupported(.enableUpcomingFeature) { - try commandLine.appendAll( - .enableUpcomingFeature, from: &parsedOptions) + if isFrontendArgSupported(.compilerAssertions) { + try commandLine.appendLast(.compilerAssertions, from: &parsedOptions) } + try commandLine.appendAll(.enableExperimentalFeature, + .disableExperimentalFeature, + .enableUpcomingFeature, + .disableUpcomingFeature, + from: &parsedOptions) + try commandLine.appendLast(.strictMemorySafety, .strictMemorySafetyMigrate, from: &parsedOptions) try commandLine.appendAll(.moduleAlias, from: &parsedOptions) if isFrontendArgSupported(.enableBareSlashRegex) { try commandLine.appendLast(.enableBareSlashRegex, from: &parsedOptions) @@ -239,6 +314,14 @@ extension Driver { if isFrontendArgSupported(.strictConcurrency) { try commandLine.appendLast(.strictConcurrency, from: &parsedOptions) } + if isFrontendArgSupported(.defaultIsolation) { + try commandLine.appendLast(.defaultIsolation, from: &parsedOptions) + } + if kind == .scanDependencies, + isFrontendArgSupported(.experimentalClangImporterDirectCc1Scan) { + try commandLine.appendAll( + .experimentalClangImporterDirectCc1Scan, from: &parsedOptions) + } // Expand the -experimental-hermetic-seal-at-link flag if parsedOptions.hasArgument(.experimentalHermeticSealAtLink) { @@ -262,16 +345,18 @@ extension Driver { } // Emit user-provided plugin paths, in order. - if isFrontendArgSupported(.externalPluginPath) { - try commandLine.appendAll(.pluginPath, .externalPluginPath, .loadPluginLibrary, .loadPluginExecutable, from: &parsedOptions) - } else if isFrontendArgSupported(.pluginPath) { - try commandLine.appendAll(.pluginPath, .loadPluginLibrary, from: &parsedOptions) + if !isPlanJobForExplicitModule { + if isFrontendArgSupported(.externalPluginPath) { + try commandLine.appendAll(.pluginPath, .externalPluginPath, .loadPluginLibrary, .loadPluginExecutable, from: &parsedOptions) + } else if isFrontendArgSupported(.pluginPath) { + try commandLine.appendAll(.pluginPath, .loadPluginLibrary, from: &parsedOptions) + } } if isFrontendArgSupported(.blockListFile) { - try Driver.findBlocklists(RelativeTo: try toolchain.executableDir).forEach { + try findBlocklists().forEach { commandLine.appendFlag(.blockListFile) - commandLine.appendPath($0) + try addPathArgument(VirtualPath.absolute($0), to: &commandLine, remap: jobNeedPathRemap) } } @@ -283,6 +368,10 @@ extension Driver { commandLine.appendFlag(ver) } + if isFrontendArgSupported(.publicModuleName) { + try commandLine.appendLast(.publicModuleName, from: &parsedOptions) + } + // Pass down -validate-clang-modules-once if we are working with a compiler that // supports it. if isFrontendArgSupported(.validateClangModulesOnce), @@ -295,19 +384,33 @@ extension Driver { try commandLine.appendLast(.enableBuiltinModule, from: &parsedOptions) } - if !useClangIncludeTree, let workingDirectory = workingDirectory { + if isFrontendArgSupported(.disableSandbox) { + try commandLine.appendLast(.disableSandbox, from: &parsedOptions) + } + + if isFrontendArgSupported(.disableDynamicActorIsolation) { + try commandLine.appendLast(.disableDynamicActorIsolation, from: &parsedOptions) + } + + if !directModuleCC1Mode, let workingDirectory = workingDirectory { // Add -Xcc -working-directory before any other -Xcc options to ensure it is // overridden by an explicit -Xcc -working-directory, although having a // different working directory is probably incorrect. commandLine.appendFlag(.Xcc) commandLine.appendFlag(.workingDirectory) commandLine.appendFlag(.Xcc) - commandLine.appendPath(.absolute(workingDirectory)) + try addPathArgument(.absolute(workingDirectory), to: &commandLine, remap: jobNeedPathRemap) } - // Resource directory. - commandLine.appendFlag(.resourceDir) - commandLine.appendPath(VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path)) + // Only pass in a resource directory to the frontend if one was passed to + // swift-driver. Make an exception for scan-dependencies jobs for now till + // we figure out a remaining problem with in-process scanning. + if parsedOptions.hasArgument(.resourceDir) || kind == .scanDependencies { + try addPathOption(option: .resourceDir, + path: VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path), + to: &commandLine, + remap: jobNeedPathRemap) + } if self.useStaticResourceDir { commandLine.appendFlag("-use-static-resource-dir") @@ -329,59 +432,112 @@ extension Driver { commandLine.appendFlag(.enableAnonymousContextMangledNames) } + // Always try to append -file-compilation-dir when debug info is used. // TODO: Should we support -fcoverage-compilation-dir? - try commandLine.appendAll(.fileCompilationDir, from: &parsedOptions) + commandLine.appendFlag(.fileCompilationDir) + if let compilationDir = parsedOptions.getLastArgument(.fileCompilationDir)?.asSingle { + let compDirPath = try VirtualPath.intern(path: compilationDir) + try addPathArgument(VirtualPath.lookup(compDirPath), to:&commandLine, remap: jobNeedPathRemap) + } else if let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory { + let compDirPath = VirtualPath.absolute(cwd) + try addPathArgument(compDirPath, to:&commandLine, remap: jobNeedPathRemap) + } } // CAS related options. - if enableCaching { + if isCachingEnabled { commandLine.appendFlag(.cacheCompileJob) - if let casPath = try getOnDiskCASPath() { + if let casPath = try Self.getOnDiskCASPath(parsedOptions: &parsedOptions, + toolchain: toolchain) { commandLine.appendFlag(.casPath) commandLine.appendFlag(casPath.pathString) } - if let pluginPath = try getCASPluginPath() { + if let pluginPath = try Self.getCASPluginPath(parsedOptions: &parsedOptions, + toolchain: toolchain) { commandLine.appendFlag(.casPluginPath) commandLine.appendFlag(pluginPath.pathString) } try commandLine.appendAll(.casPluginOption, from: &parsedOptions) try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) + if !useClangIncludeTree { + commandLine.appendFlag(.noClangIncludeTree) + } } - if useClangIncludeTree { - commandLine.appendFlag(.clangIncludeTree) - } + addCacheReplayMapping(to: &commandLine) // Pass through any subsystem flags. try commandLine.appendAll(.Xllvm, from: &parsedOptions) - if !useClangIncludeTree { + // Pass through all -Xcc flags if not under directModuleCC1Mode. + if !directModuleCC1Mode { try commandLine.appendAll(.Xcc, from: &parsedOptions) } - if let importedObjCHeader = importedObjCHeader, - bridgingHeaderHandling != .ignored { - commandLine.appendFlag(.importObjcHeader) - if bridgingHeaderHandling == .precompiled, - let pch = bridgingPrecompiledHeader { - // For explicit module build, we directly pass the compiled pch as - // `-import-objc-header`, rather than rely on swift-frontend to locate + let objcHeaderFile = (kind == .scanDependencies) ? originalObjCHeaderFile : importedObjCHeader + if let importedObjCHeader = objcHeaderFile, bridgingHeaderHandling != .ignored { + if bridgingHeaderHandling == .precompiled, let pch = bridgingPrecompiledHeader { + // For explicit module build, we directly pass the compiled pch to + // swift-frontend, rather than rely on swift-frontend to locate // the pch in the pchOutputDir and can start an implicit build in case // of a lookup failure. if parsedOptions.contains(.pchOutputDir) && !parsedOptions.contains(.driverExplicitModuleBuild) { - commandLine.appendPath(VirtualPath.lookup(importedObjCHeader)) + commandLine.appendFlag(.importObjcHeader) + try addPathArgument(VirtualPath.lookup(importedObjCHeader), to:&commandLine, remap: jobNeedPathRemap) try commandLine.appendLast(.pchOutputDir, from: &parsedOptions) if !compilerMode.isSingleCompilation { commandLine.appendFlag(.pchDisableValidation) } } else { - commandLine.appendPath(VirtualPath.lookup(pch)) + // If header chaining is enabled, pass objc header through `-import-objc-header` and + // PCH file through `-import-pch`. Otherwise, pass either the PCH or header through + // `-import-objc-header` option. + if isFrontendArgSupported(.importPch), importedObjCHeader != originalObjCHeaderFile { + commandLine.appendFlag(.importPch) + try addPathArgument(VirtualPath.lookup(pch), to:&commandLine, remap: jobNeedPathRemap) + if let originalHeader = originalObjCHeaderFile { + commandLine.appendFlag(.importObjcHeader) + try addPathArgument(VirtualPath.lookup(originalHeader), to:&commandLine, remap: jobNeedPathRemap) + } + } else { + commandLine.appendFlag(.importObjcHeader) + try addPathArgument(VirtualPath.lookup(pch), to:&commandLine, remap: jobNeedPathRemap) + } } } else { - commandLine.appendPath(VirtualPath.lookup(importedObjCHeader)) + commandLine.appendFlag(.importObjcHeader) + try addPathArgument(VirtualPath.lookup(importedObjCHeader), to:&commandLine, remap: jobNeedPathRemap) + } + } + + // Pass bridging header chaining options. + if isFrontendArgSupported(.autoBridgingHeaderChaining) { + if bridgingHeaderChaining { + commandLine.appendFlag(.autoBridgingHeaderChaining) + } else { + commandLine.appendFlag(.noAutoBridgingHeaderChaining) + } + } + + if parsedOptions.contains(.enableDeterministicCheck), + isFrontendArgSupported(.enableDeterministicCheck) { + commandLine.appendFlag(.enableDeterministicCheck) + commandLine.appendFlag(.alwaysCompileOutputFiles) + if enableCaching { + commandLine.appendFlag(.cacheDisableReplay) } } + // Pass along -no-verify-emitted-module-interface only if it's effective. + // Assume verification by default as we want to know only when the user skips + // the verification. + if isFrontendArgSupported(.noVerifyEmittedModuleInterface) && + !parsedOptions.hasFlag(positive: .verifyEmittedModuleInterface, + negative: .noVerifyEmittedModuleInterface, + default: true) { + commandLine.appendFlag("-no-verify-emitted-module-interface") + } + // Repl Jobs shouldn't include -module-name. if compilerMode != .repl && compilerMode != .intro { commandLine.appendFlags("-module-name", moduleOutputInfo.name) @@ -396,38 +552,108 @@ extension Driver { commandLine.appendFlag("-frontend-parseable-output") } + // If explicit auto-linking is enabled, ensure that compiler tasks do not produce + // auto-link load commands in resulting object files. + if parsedOptions.hasArgument(.explicitAutoLinking) { + commandLine.appendFlag(.disableAllAutolinking) + } + savedUnknownDriverFlagsForSwiftFrontend.forEach { commandLine.appendFlag($0) } + let toolchainStdlibPath = VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path) + .appending(components: frontendTargetInfo.target.triple.platformName() ?? "", "Swift.swiftmodule") + let hasToolchainStdlib = try fileSystem.exists(toolchainStdlibPath) + + let skipMacroOptions = isPlanJobForExplicitModule && isFrontendArgSupported(.loadResolvedPlugin) + // If the resource directory has the standard library, prefer the toolchain's plugins + // to the platform SDK plugins. + // For explicit module build, the resolved plugins are provided by scanner. + if hasToolchainStdlib, !skipMacroOptions { + try addPluginPathArguments(commandLine: &commandLine) + } + try toolchain.addPlatformSpecificCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, frontendTargetInfo: frontendTargetInfo, - driver: &self) + driver: &self, + skipMacroOptions: skipMacroOptions) + + // Otherwise, prefer the platform's plugins. + if !hasToolchainStdlib, !skipMacroOptions { + try addPluginPathArguments(commandLine: &commandLine) + } + + if let passPluginPath = parsedOptions.getLastArgument(.loadPassPluginEQ), + isFrontendArgSupported(.loadPassPluginEQ) { + commandLine.appendFlag("-load-pass-plugin=\(passPluginPath.asSingle)") + } + } + + mutating func addRuntimeLibraryFlags(commandLine: inout [Job.ArgTemplate]) throws { + guard targetTriple.isWindows else { return } + + enum RuntimeFlavour { + case MT + case MTd + case MD + case MDd + } + + let option = parsedOptions.getLastArgument(.libc) + + // NOTE: default to `/MD`. This is different from `cl`'s default behaviour + // of `/MT` on the command line, however, Visual Studio 2015 and newer will + // default `/MD` as well. Furthermore, this is far more useful of a mode + // since the `/MT` mode requires that everything is statically linked. + let runtime: RuntimeFlavour? = switch (option?.asSingle ?? "MD") { + case "MD", "MultiThreadedDLL", "shared-ucrt": + .MD + case "MDd", "MultiThreadedDebugDLL", "shared-debug-ucrt": + .MDd + case "MT", "MultiThreaded", "static-ucrt": + .MT + case "MTd", "MultiThreadedDebug", "static-debug-ucrt": + .MTd + default: + nil + } + + guard let runtime else { + diagnosticEngine.emit(.error_invalid_arg_value(arg: .libc, value: option!.asSingle)) + return + } - // Platform-agnostic options that need to happen after platform-specific ones. - if isFrontendArgSupported(.pluginPath) { - // Default paths for compiler plugins found within the toolchain - // (loaded as shared libraries). - let pluginPathRoot = VirtualPath.absolute(try toolchain.executableDir.parentDirectory) - commandLine.appendFlag(.pluginPath) - commandLine.appendPath(pluginPathRoot.pluginPath) + commandLine.appendFlag(.autolinkLibrary) + commandLine.appendFlag("oldnames") - commandLine.appendFlag(.pluginPath) - commandLine.appendPath(pluginPathRoot.localPluginPath) + commandLine.appendFlag(.autolinkLibrary) + let name = switch (runtime) { + case .MD: "msvcrt" + case .MDd: "msvcrtd" + case .MT: "libcmt" + case .MTd: "libcmtd" + } + commandLine.appendFlag(name) + + commandLine.appendFlag(.Xcc) + commandLine.appendFlag("-D_MT") + + if [.MD, .MDd].contains(runtime) { + commandLine.appendFlag(.Xcc) + commandLine.appendFlag("-D_DLL") } } - func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate], - pchCompileJob: Job?) throws { - guard let pchJob = pchCompileJob, enableCaching else { return } + mutating func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate], + pchCompileJob: Job?) throws { + guard let pchJob = pchCompileJob, isCachingEnabled else { return } - // The pch input file (the bridging header) is added as last inputs to the job. - guard let inputFile = pchJob.inputs.last else { assertionFailure("no input files from pch job"); return } - assert(inputFile.type == .objcHeader, "Expect objc header input type") - let bridgingHeaderCacheKey = try interModuleDependencyOracle.computeCacheKeyForOutput(kind: .pch, - commandLine: pchJob.commandLine, - input: inputFile.fileHandle) + assert(pchJob.outputCacheKeys.count == 1, "Expect one and only one cache key from pch job") + guard let bridgingHeaderCacheKey = pchJob.outputCacheKeys.first?.value else { + fatalError("pch job doesn't have an associated cache key") + } commandLine.appendFlag("-bridging-header-pch-key") commandLine.appendFlag(bridgingHeaderCacheKey) } @@ -436,6 +662,8 @@ extension Driver { primaryInputs: [TypedVirtualPath], inputsGeneratingCodeCount: Int, inputOutputMap: inout [TypedVirtualPath: [TypedVirtualPath]], + moduleOutputInfo: ModuleOutputInfo, + moduleOutputPaths: SupplementalModuleTargetOutputPaths, includeModuleTracePath: Bool, indexFilePath: TypedVirtualPath?) throws -> [TypedVirtualPath] { var flaggedInputOutputPairs: [(flag: String, input: TypedVirtualPath?, output: TypedVirtualPath)] = [] @@ -464,7 +692,7 @@ extension Driver { // Alongside primary output outputPath = try output.file.replacingExtension(with: outputType).intern() } else { - outputPath = VirtualPath.createUniqueTemporaryFile(RelativePath(input.file.basenameWithoutExt.appendingFileTypeExtension(outputType))).intern() + outputPath = try VirtualPath.createUniqueTemporaryFile(RelativePath(validating: input.file.basenameWithoutExt.appendingFileTypeExtension(outputType))).intern() } // Update the input-output file map. @@ -482,7 +710,9 @@ extension Driver { } /// Add all of the outputs needed for a given input. - func addAllOutputsFor(input: TypedVirtualPath?) throws { + func addAllOutputsFor(input: TypedVirtualPath?, + moduleOutputInfo: ModuleOutputInfo, + moduleOutputPaths: SupplementalModuleTargetOutputPaths) throws { if !emitModuleSeparately { // Generate the module files with the main job. try addOutputOfType( @@ -492,12 +722,12 @@ extension Driver { flag: "-emit-module-path") try addOutputOfType( outputType: .swiftDocumentation, - finalOutputPath: moduleDocOutputPath, + finalOutputPath: moduleOutputPaths.moduleDocOutputPath, input: input, flag: "-emit-module-doc-path") try addOutputOfType( outputType: .swiftSourceInfoFile, - finalOutputPath: moduleSourceInfoPath, + finalOutputPath: moduleOutputPaths.moduleSourceInfoPath, input: input, flag: "-emit-module-source-info-path") } @@ -535,10 +765,14 @@ extension Driver { if compilerMode.usesPrimaryFileInputs { for input in primaryInputs { - try addAllOutputsFor(input: input) + try addAllOutputsFor(input: input, + moduleOutputInfo: moduleOutputInfo, + moduleOutputPaths: moduleOutputPaths) } } else { - try addAllOutputsFor(input: nil) + try addAllOutputsFor(input: nil, + moduleOutputInfo: moduleOutputInfo, + moduleOutputPaths: moduleOutputPaths) if !emitModuleSeparately { // Outputs that only make sense when the whole module is processed @@ -551,28 +785,41 @@ extension Driver { try addOutputOfType( outputType: .swiftInterface, - finalOutputPath: swiftInterfacePath, + finalOutputPath: moduleOutputPaths.swiftInterfacePath, input: nil, flag: "-emit-module-interface-path") try addOutputOfType( outputType: .privateSwiftInterface, - finalOutputPath: swiftPrivateInterfacePath, + finalOutputPath: moduleOutputPaths.swiftPrivateInterfacePath, input: nil, flag: "-emit-private-module-interface-path") + if let pkgName = packageName, !pkgName.isEmpty { + try addOutputOfType( + outputType: .packageSwiftInterface, + finalOutputPath: moduleOutputPaths.swiftPackageInterfacePath, + input: nil, + flag: "-emit-package-module-interface-path") + } try addOutputOfType( outputType: .tbd, finalOutputPath: tbdPath, input: nil, flag: "-emit-tbd-path") - if let abiDescriptorPath = abiDescriptorPath { + if let abiDescriptorPath = moduleOutputPaths.abiDescriptorFilePath { try addOutputOfType(outputType: .jsonABIBaseline, finalOutputPath: abiDescriptorPath.fileHandle, input: nil, flag: "-emit-abi-descriptor-path") } + + try addOutputOfType( + outputType: .jsonAPIDescriptor, + finalOutputPath: moduleOutputPaths.apiDescriptorFilePath, + input: nil, + flag: "-emit-api-descriptor-path") } } @@ -591,7 +838,7 @@ extension Driver { remapOutputPath = try output.file.replacingExtension(with: .remap) } else { remapOutputPath = - VirtualPath.createUniqueTemporaryFile(RelativePath(input.file.basenameWithoutExt.appendingFileTypeExtension(.remap))) + try VirtualPath.createUniqueTemporaryFile(RelativePath(validating: input.file.basenameWithoutExt.appendingFileTypeExtension(.remap))) } flaggedInputOutputPairs.append((flag: "-emit-remap-file-path", @@ -609,7 +856,7 @@ extension Driver { var entries = [VirtualPath.Handle: [FileType: VirtualPath.Handle]]() for input in primaryInputs { if let output = inputOutputMap[input]?.first { - addEntry(&entries, input: input, output: output) + try addEntry(&entries, input: input, output: output) } else { // Primary inputs are expected to appear in the output file map even // if they have no corresponding outputs. @@ -628,7 +875,7 @@ extension Driver { } for flaggedPair in flaggedInputOutputPairs { - addEntry(&entries, input: flaggedPair.input, output: flaggedPair.output) + try addEntry(&entries, input: flaggedPair.input, output: flaggedPair.output) } // To match the legacy driver behavior, make sure we add an entry for the // file under indexing and the primary output file path. @@ -636,8 +883,8 @@ extension Driver { entries[indexFilePath.fileHandle] = [.indexData: idxOutput.fileHandle] } let outputFileMap = OutputFileMap(entries: entries) - let fileList = VirtualPath.createUniqueFilelist(RelativePath("supplementaryOutputs"), - .outputFileMap(outputFileMap)) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "supplementaryOutputs"), + .outputFileMap(outputFileMap)) commandLine.appendFlag(.supplementaryOutputFileMap) commandLine.appendPath(fileList) } else { @@ -662,42 +909,62 @@ extension Driver { try commandLine.appendLast(.symbolGraphMinimumAccessLevel, from: &parsedOptions) } - func addEntry(_ entries: inout [VirtualPath.Handle: [FileType: VirtualPath.Handle]], input: TypedVirtualPath?, output: TypedVirtualPath) { + mutating func addEntry(_ entries: inout [VirtualPath.Handle: [FileType: VirtualPath.Handle]], input: TypedVirtualPath?, output: TypedVirtualPath) throws { let entryInput: VirtualPath.Handle if let input = input?.fileHandle, input != OutputFileMap.singleInputKey { entryInput = input } else { - entryInput = inputFiles[0].fileHandle + guard let firstSourceInputHandle = inputFiles.first(where:{ $0.type == .swift })?.fileHandle else { + fatalError("Formulating swift-frontend invocation without any input .swift files") + } + entryInput = firstSourceInputHandle } - entries[entryInput, default: [:]][output.type] = output.fileHandle + let inputEntry = isCachingEnabled ? remapPath(VirtualPath.lookup(entryInput)).intern() : entryInput + entries[inputEntry, default: [:]][output.type] = output.fileHandle } /// Adds all dependencies required for an explicit module build /// to inputs and command line arguments of a compile job. - func addExplicitModuleBuildArguments(inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { - guard var dependencyPlanner = explicitDependencyBuildPlanner else { - fatalError("No dependency planner in Explicit Module Build mode.") - } - try dependencyPlanner.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine) + mutating func addExplicitModuleBuildArguments(inputs: inout [TypedVirtualPath], + commandLine: inout [Job.ArgTemplate]) throws { + try explicitDependencyBuildPlanner?.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine) } /// Adds all dependencies required for an explicit module build of the bridging header /// to inputs and command line arguments of a compile job. - func addExplicitPCHBuildArguments(inputs: inout [TypedVirtualPath], - commandLine: inout [Job.ArgTemplate]) throws { - guard var dependencyPlanner = explicitDependencyBuildPlanner else { - fatalError("No dependency planner in Explicit Module Build mode.") + mutating func addExplicitPCHBuildArguments(inputs: inout [TypedVirtualPath], + commandLine: inout [Job.ArgTemplate]) throws { + try explicitDependencyBuildPlanner?.resolveBridgingHeaderDependencies(inputs: &inputs, commandLine: &commandLine) + } + + mutating func addPluginPathArguments(commandLine: inout [Job.ArgTemplate]) throws { + guard isFrontendArgSupported(.pluginPath) else { + return + } + let pluginPathRoot = VirtualPath.absolute(try toolchain.executableDir.parentDirectory) + + if isFrontendArgSupported(.inProcessPluginServerPath) { + commandLine.appendFlag(.inProcessPluginServerPath) +#if os(Windows) + commandLine.appendPath(pluginPathRoot.appending(components: "bin", sharedLibraryName("SwiftInProcPluginServer"))) +#else + commandLine.appendPath(pluginPathRoot.appending(components: "lib", "swift", "host", sharedLibraryName("libSwiftInProcPluginServer"))) +#endif } - try dependencyPlanner.resolveBridgingHeaderDependencies(inputs: &inputs, commandLine: &commandLine) + + // Default paths for compiler plugins found within the toolchain + // (loaded as shared libraries). + commandLine.appendFlag(.pluginPath) + commandLine.appendPath(pluginPathRoot.pluginPath) + + commandLine.appendFlag(.pluginPath) + commandLine.appendPath(pluginPathRoot.localPluginPath) } + /// If explicit dependency planner supports creating bridging header pch command. - public func supportsBridgingHeaderPCHCommand() throws -> Bool { - guard let dependencyPlanner = explicitDependencyBuildPlanner else { - return false - } - return try dependencyPlanner.supportsBridgingHeaderPCHCommand() + public var supportsBridgingHeaderPCHCommand: Bool { + return explicitDependencyBuildPlanner?.supportsBridgingHeaderPCHCommand ?? false } /// In Explicit Module Build mode, distinguish between main module jobs and intermediate dependency build jobs, @@ -706,3 +973,126 @@ extension Driver { return job.moduleName == moduleOutputInfo.name } } + +extension Driver { + private func getAbsolutePathFromVirtualPath(_ path: VirtualPath) -> AbsolutePath? { + guard let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory else { + return nil + } + return path.resolvedRelativePath(base: cwd).absolutePath + } + + private mutating func remapPath(absolute path: AbsolutePath) -> AbsolutePath { + guard !prefixMapping.isEmpty else { + return path + } + for (prefix, value) in prefixMapping { + if path.isDescendantOfOrEqual(to: prefix) { + return value.appending(path.relative(to: prefix)) + } + } + return path + } + + public mutating func remapPath(_ path: VirtualPath) -> VirtualPath { + guard !prefixMapping.isEmpty, + let absPath = getAbsolutePathFromVirtualPath(path) else { + return path + } + let mappedPath = remapPath(absolute: absPath) + return try! VirtualPath(path: mappedPath.pathString) + } + + /// Helper function to add path to commandLine. Function will validate the path, and remap the path if needed. + public mutating func addPathArgument(_ path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws { + guard remap && isCachingEnabled else { + commandLine.appendPath(path) + return + } + let mappedPath = remapPath(path) + commandLine.appendPath(mappedPath) + } + + public mutating func addPathOption(_ option: ParsedOption, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws { + let path = try VirtualPath(path: option.argument.asSingle) + try addPathOption(option: option.option, path: path, to: &commandLine, remap: remap) + } + + public mutating func addPathOption(option: Option, path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws { + let needRemap = remap && isCachingEnabled && option.attributes.contains(.argumentIsPath) && + !option.attributes.contains(.cacheInvariant) + let commandPath = needRemap ? remapPath(path) : path + if option.kind == .joined { + commandLine.append(.joinedOptionAndPath(option.spelling, commandPath)) + } else { + // All other kinds that involves a path can be added as separated args. + commandLine.appendFlag(option) + commandLine.appendPath(commandPath) + } + } + + /// Helper function to add last argument with path to command-line. + public mutating func addLastArgumentWithPath(_ options: Option..., + to commandLine: inout [Job.ArgTemplate], + remap: Bool = true) throws { + guard let parsedOption = parsedOptions.last(for: options) else { + return + } + try addPathOption(parsedOption, to: &commandLine, remap: remap) + } + + /// Helper function to add all arguments with path to command-line. + public mutating func addAllArgumentsWithPath(_ options: Option..., + to commandLine: inout [Job.ArgTemplate], + remap: Bool) throws { + for matching in parsedOptions.arguments(for: options) { + try addPathOption(matching, to: &commandLine, remap: remap) + } + } + + public mutating func addCacheReplayMapping(to commandLine: inout [Job.ArgTemplate]) { + if isCachingEnabled && isFrontendArgSupported(.scannerPrefixMap) { + for (key, value) in prefixMapping { + commandLine.appendFlag("-cache-replay-prefix-map") + commandLine.appendFlag(value.pathString + "=" + key.pathString) + } + } + } +} + +extension Driver { + public mutating func computeOutputCacheKeyForJob(commandLine: [Job.ArgTemplate], + inputs: [(TypedVirtualPath, Int)]) throws -> [TypedVirtualPath: String] { + // No caching setup, return empty dictionary. + guard let cas = self.cas else { + return [:] + } + // Resolve command-line first. + let arguments: [String] = try executor.resolver.resolveArgumentList(for: commandLine) + + return try inputs.reduce(into: [:]) { keys, input in + keys[input.0] = try cas.computeCacheKey(commandLine: arguments, index: input.1) + } + } + + public mutating func computeOutputCacheKey(commandLine: [Job.ArgTemplate], + index: Int) throws -> String? { + // No caching setup, return empty dictionary. + guard let cas = self.cas else { + return nil + } + // Resolve command-line first. + let arguments: [String] = try executor.resolver.resolveArgumentList(for: commandLine) + return try cas.computeCacheKey(commandLine: arguments, index: index) + } +} + +extension ParsedOptions { + /// Checks whether experimental embedded mode is enabled. + var isEmbeddedEnabled: Bool { + mutating get { + let experimentalFeatures = self.arguments(for: .enableExperimentalFeature) + return experimentalFeatures.map(\.argument).map(\.asSingle).contains("Embedded") + } + } +} diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index 5a4e8fd8b..a4ff2943d 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -30,11 +30,13 @@ extension Driver { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - if try supportsBridgingHeaderPCHCommand() { + if supportsBridgingHeaderPCHCommand { try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine) + addCacheReplayMapping(to: &commandLine) } else { try addGeneratePCHFlags(commandLine: &commandLine, inputs: &inputs) } + try addRuntimeLibraryFlags(commandLine: &commandLine) // TODO: Should this just be pch output with extension changed? if parsedOptions.hasArgument(.serializeDiagnostics), let outputDirectory = parsedOptions.getLastArgument(.pchOutputDir)?.asSingle { @@ -49,8 +51,8 @@ extension Driver { path = try VirtualPath(path: outputDirectory).appending(component: outputName.appendingFileTypeExtension(.diagnostics)) } else { path = - VirtualPath.createUniqueTemporaryFile( - RelativePath(input.file.basenameWithoutExt.appendingFileTypeExtension(.diagnostics))) + try VirtualPath.createUniqueTemporaryFile( + RelativePath(validating: input.file.basenameWithoutExt.appendingFileTypeExtension(.diagnostics))) } commandLine.appendPath(path) outputs.append(.init(file: path.intern(), type: .diagnostics)) @@ -67,7 +69,9 @@ extension Driver { outputs.append(output) inputs.append(input) - commandLine.appendPath(input.file) + try addPathArgument(input.file, to: &commandLine) + + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(input, 0)]) return Job( moduleName: moduleOutputInfo.name, @@ -77,7 +81,8 @@ extension Driver { displayInputs: [], inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys ) } } diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index 86f1a4543..6e25aa72b 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -51,6 +51,7 @@ extension Driver { commandLine: &commandLine, inputs: &inputs, kind: .generatePCM, bridgingHeaderHandling: .ignored) try commandLine.appendLast(.indexStorePath, from: &parsedOptions) + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(input, 0)]) return Job( moduleName: moduleOutputInfo.name, @@ -60,7 +61,8 @@ extension Driver { displayInputs: [], inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys ) } diff --git a/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift index 8030f9035..fa2bd8132 100644 --- a/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift @@ -16,31 +16,12 @@ import func TSCBasic.lookupExecutablePath import struct TSCBasic.AbsolutePath extension GenericUnixToolchain { - private func defaultLinker(for targetTriple: Triple) -> String? { - if targetTriple.os == .openbsd || targetTriple.os == .freeBSD || - targetTriple.environment == .android { - return "lld" - } - - switch targetTriple.arch { - case .arm, .aarch64, .armeb, .thumb, .thumbeb: - // BFD linker has issues wrt relocation of the protocol conformance - // section on these targets, it also generates COPY relocations for - // final executables, as such, unless specified, we default to gold - // linker. - return "gold" - case .x86, .x86_64, .ppc64, .ppc64le, .systemz: - // BFD linker has issues wrt relocations against protected symbols. - return "gold" - default: - // Otherwise, use the default BFD linker. - return "" - } - } - private func majorArchitectureName(for triple: Triple) -> String { // The concept of a "major" arch name only applies to Linux triples - guard triple.os == .linux else { return triple.archName } + // We change spellings for amd64/x86_64 for OpenBSD here too to match LLVM. + guard triple.os == .linux || triple.os == .openbsd else { return triple.archName } + + if triple.os == .openbsd && triple.archName == "amd64" { return "x86_64" } // HACK: We don't wrap LLVM's ARM target architecture parsing, and we should // definitely not try to port it. This check was only normalizing @@ -70,39 +51,17 @@ extension GenericUnixToolchain { commandLine.appendFlag("-shared") fallthrough case .executable: - // Select the linker to use. - var linker: String? - if let arg = parsedOptions.getLastArgument(.useLd) { - linker = arg.asSingle + // Select the linker to use. + if let arg = parsedOptions.getLastArgument(.useLd)?.asSingle { + commandLine.appendFlag("-fuse-ld=\(arg)") } else if lto != nil { - linker = "lld" - } else { - linker = defaultLinker(for: targetTriple) + commandLine.appendFlag("-fuse-ld=lld") } - if let linker = linker { - #if os(Haiku) - // For now, passing -fuse-ld on Haiku doesn't work as swiftc doesn't - // recognise it. Passing -use-ld= as the argument works fine. - commandLine.appendFlag("-use-ld=\(linker)") - #else - commandLine.appendFlag("-fuse-ld=\(linker)") - #endif - // Starting with lld 13, Swift stopped working with the lld - // --gc-sections implementation for ELF, unless -z nostart-stop-gc is - // also passed to lld: - // - // https://reviews.llvm.org/D96914 - if linker == "lld" || linker.hasSuffix("ld.lld") { - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag("-z") - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag("nostart-stop-gc") - } + if let arg = parsedOptions.getLastArgument(.ldPath)?.asSingle { + commandLine.append(.joinedOptionAndPath("--ld-path=", try VirtualPath(path: arg))) } - try commandLine.appendLast(.ldPath, from: &parsedOptions) - // Configure the toolchain. // // By default use the system `clang` to perform the link. We use `clang` for @@ -122,11 +81,17 @@ extension GenericUnixToolchain { // just using `clang` and avoid a dependency on the C++ runtime. var cxxCompatEnabled = parsedOptions.hasArgument(.enableExperimentalCxxInterop) if let cxxInteropMode = parsedOptions.getLastArgument(.cxxInteroperabilityMode) { - if cxxInteropMode.asSingle == "swift-5.9" { + if cxxInteropMode.asSingle != "off" { cxxCompatEnabled = true } } + let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, + negative: .noStaticStdlib, + default: false) + let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, + negative: .noStaticExecutable, + default: false) let clangTool: Tool = cxxCompatEnabled ? .clangxx : .clang var clangPath = try getToolPath(clangTool) if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) { @@ -146,21 +111,40 @@ extension GenericUnixToolchain { } // Executables on Linux get -pie - if targetTriple.os == .linux && linkerOutputType == .executable { + if targetTriple.os == .linux && linkerOutputType == .executable && !staticExecutable { commandLine.appendFlag("-pie") } - let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, - negative: .noStaticStdlib, - default: false) - let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, - negative: .noStaticExecutable, - default: false) + // On some platforms we want to enable --build-id + if targetTriple.os == .linux + || targetTriple.os == .freeBSD + || targetTriple.os == .openbsd + || parsedOptions.hasArgument(.buildId) { + commandLine.appendFlag("-Xlinker") + if let buildId = parsedOptions.getLastArgument(.buildId)?.asSingle { + commandLine.appendFlag("--build-id=\(buildId)") + } else { + commandLine.appendFlag("--build-id") + } + } + + if targetTriple.os == .openbsd && targetTriple.arch == .aarch64 { + let btcfiEnabled = targetInfo.target.openbsdBTCFIEnabled ?? false + if !btcfiEnabled { + commandLine.appendFlag("-Xlinker") + commandLine.appendFlag("-z") + commandLine.appendFlag("-Xlinker") + commandLine.appendFlag("nobtcfi") + } + } + + let isEmbeddedEnabled = parsedOptions.isEmbeddedEnabled + let toolchainStdlibRpath = parsedOptions .hasFlag(positive: .toolchainStdlibRpath, negative: .noToolchainStdlibRpath, default: true) - let hasRuntimeArgs = !(staticStdlib || staticExecutable) + let hasRuntimeArgs = !(staticStdlib || staticExecutable || isEmbeddedEnabled) let runtimePaths = try runtimeLibraryPaths( for: targetInfo, @@ -169,8 +153,16 @@ extension GenericUnixToolchain { isShared: hasRuntimeArgs ) - if hasRuntimeArgs && targetTriple.environment != .android && - toolchainStdlibRpath { + // An exception is made for native Android environments like the Termux + // app as they build and run natively like a Unix environment on Android, + // so add the stdlib RPATH by default there. + #if os(Android) + let addRpath = true + #else + let addRpath = targetTriple.environment != .android + #endif + + if hasRuntimeArgs && addRpath && toolchainStdlibRpath { // FIXME: We probably shouldn't be adding an rpath here unless we know // ahead of time the standard library won't be copied. for path in runtimePaths { @@ -181,14 +173,30 @@ extension GenericUnixToolchain { } } - if !parsedOptions.hasArgument(.nostartfiles) { - let swiftrtPath = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) - .appending( - components: targetTriple.platformName() ?? "", - String(majorArchitectureName(for: targetTriple)), - "swiftrt.o" - ) - commandLine.appendPath(swiftrtPath) + if targetInfo.sdkPath != nil { + for libpath in targetInfo.runtimeLibraryImportPaths { + commandLine.appendFlag(.L) + commandLine.appendPath(VirtualPath.lookup(libpath.path)) + } + } + + if !isEmbeddedEnabled && !parsedOptions.hasArgument(.nostartfiles) { + let rsrc: VirtualPath + // Prefer the swiftrt.o runtime file from the SDK if it's specified. + if let sdk = targetInfo.sdkPath { + let swiftDir: String + if staticStdlib || staticExecutable { + swiftDir = "swift_static" + } else { + swiftDir = "swift" + } + rsrc = VirtualPath.lookup(sdk.path).appending(components: "usr", "lib", swiftDir) + } else { + rsrc = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) + } + let platform: String = targetTriple.platformName() ?? "" + let architecture: String = majorArchitectureName(for: targetTriple) + commandLine.appendPath(rsrc.appending(components: platform, architecture, "swiftrt.o")) } // If we are linking statically, we need to add all @@ -228,7 +236,15 @@ extension GenericUnixToolchain { commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) } - if let path = targetInfo.sdkPath?.path { + if let sysroot = parsedOptions.getLastArgument(.sysroot)?.asSingle { + commandLine.appendFlag("--sysroot") + try commandLine.appendPath(VirtualPath(path: sysroot)) + } else if targetTriple.environment == .android, + let sysroot = AndroidNDK.getDefaultSysrootPath(in: self.env) + { + commandLine.appendFlag("--sysroot") + try commandLine.appendPath(VirtualPath(path: sysroot.pathString)) + } else if let path = targetInfo.sdkPath?.path { commandLine.appendFlag("--sysroot") commandLine.appendPath(VirtualPath.lookup(path)) } @@ -251,7 +267,9 @@ extension GenericUnixToolchain { linkFilePath = linkFilePath?.appending(component: "static-stdlib-args.lnk") } else { linkFilePath = nil - commandLine.appendFlag("-lswiftCore") + if !isEmbeddedEnabled { + commandLine.appendFlag("-lswiftCore") + } } if let linkFile = linkFilePath { @@ -286,9 +304,10 @@ extension GenericUnixToolchain { } if parsedOptions.hasArgument(.profileGenerate) { + let environment = (targetTriple.environment == .android) ? "-android" : "" let libProfile = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) - .appending(components: "clang", "lib", targetTriple.osName, - "libclang_rt.profile-\(targetTriple.archName).a") + .appending(components: "clang", "lib", targetTriple.osNameUnversioned, + "libclang_rt.profile-\(targetTriple.archName)\(environment).a") commandLine.appendPath(libProfile) // HACK: Hard-coded from llvm::getInstrProfRuntimeHookVarName() @@ -315,13 +334,7 @@ extension GenericUnixToolchain { from: &parsedOptions ) addLinkedLibArgs(to: &commandLine, parsedOptions: &parsedOptions) - // Because we invoke `clang` as the linker executable, we must still - // use `-Xlinker` for linker-specific arguments. - for linkerOpt in parsedOptions.arguments(for: .Xlinker) { - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag(linkerOpt.argument.asSingle) - } - try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + try addExtraClangLinkerArgs(to: &commandLine, parsedOptions: &parsedOptions) // This should be the last option, for convenience in checking output. commandLine.appendFlag(.o) diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift index 7aab9aceb..73508c1fd 100644 --- a/Sources/SwiftDriver/Jobs/InterpretJob.swift +++ b/Sources/SwiftDriver/Jobs/InterpretJob.swift @@ -28,7 +28,7 @@ extension Driver { } try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .interpret) - // FIXME: MSVC runtime flags + try addRuntimeLibraryFlags(commandLine: &commandLine) try commandLine.appendLast(.parseSil, from: &parsedOptions) toolchain.addLinkedLibArgs(to: &commandLine, parsedOptions: &parsedOptions) diff --git a/Sources/SwiftDriver/Jobs/Job.swift b/Sources/SwiftDriver/Jobs/Job.swift index 0c3571165..2b2f5c370 100644 --- a/Sources/SwiftDriver/Jobs/Job.swift +++ b/Sources/SwiftDriver/Jobs/Job.swift @@ -43,6 +43,7 @@ public struct Job: Codable, Equatable, Hashable { case generateABIBaseline = "generate-abi-baseline" case compareAPIBaseline = "compare-api-baseline" case compareABIBaseline = "Check ABI stability" + case printSupportedFeatures = "print-supported-features" } public enum ArgTemplate: Equatable, Hashable { @@ -101,6 +102,9 @@ public struct Job: Codable, Equatable, Hashable { /// The kind of job. public var kind: Kind + /// The Cache Key for the compilation. It is a dictionary from input file to its output cache key. + public var outputCacheKeys: [TypedVirtualPath: String] + /// A map from a primary input to all of its corresponding outputs private var compileInputOutputMap: [TypedVirtualPath : [TypedVirtualPath]] @@ -113,6 +117,7 @@ public struct Job: Codable, Equatable, Hashable { inputs: [TypedVirtualPath], primaryInputs: [TypedVirtualPath], outputs: [TypedVirtualPath], + outputCacheKeys: [TypedVirtualPath: String] = [:], inputOutputMap: [TypedVirtualPath : [TypedVirtualPath]] = [:], extraEnvironment: [String: String] = [:], requiresInPlaceExecution: Bool = false @@ -125,6 +130,7 @@ public struct Job: Codable, Equatable, Hashable { self.inputs = inputs self.primaryInputs = primaryInputs self.outputs = outputs + self.outputCacheKeys = outputCacheKeys self.compileInputOutputMap = inputOutputMap self.extraEnvironment = extraEnvironment self.requiresInPlaceExecution = requiresInPlaceExecution @@ -246,6 +252,9 @@ extension Job : CustomStringConvertible { case .compareABIBaseline: return "Comparing ABI of \(moduleName) to baseline" + + case .printSupportedFeatures: + return "Print supported upcoming and experimental features" } } @@ -265,7 +274,7 @@ extension Job.Kind { switch self { case .backend, .compile, .mergeModule, .emitModule, .compileModuleFromInterface, .generatePCH, .generatePCM, .dumpPCM, .interpret, .repl, .printTargetInfo, - .versionRequest, .emitSupportedFeatures, .scanDependencies, .verifyModuleInterface: + .versionRequest, .emitSupportedFeatures, .scanDependencies, .verifyModuleInterface, .printSupportedFeatures: return true case .autolinkExtract, .generateDSYM, .help, .link, .verifyDebugInfo, .moduleWrap, @@ -285,7 +294,22 @@ extension Job.Kind { .help, .link, .verifyDebugInfo, .scanDependencies, .emitSupportedFeatures, .moduleWrap, .verifyModuleInterface, .generateAPIBaseline, .generateABIBaseline, .compareAPIBaseline, - .compareABIBaseline: + .compareABIBaseline, .printSupportedFeatures: + return false + } + } + + /// Whether this job supports caching. + public var supportCaching: Bool { + switch self { + case .compile, .emitModule, .generatePCH, .compileModuleFromInterface, + .generatePCM, .verifyModuleInterface: + return true + case .backend, .mergeModule, .dumpPCM, .interpret, .repl, .printTargetInfo, + .versionRequest, .autolinkExtract, .generateDSYM, .help, .link, + .verifyDebugInfo, .scanDependencies, .emitSupportedFeatures, .moduleWrap, + .generateAPIBaseline, .generateABIBaseline, .compareAPIBaseline, + .compareABIBaseline, .printSupportedFeatures: return false } } diff --git a/Sources/SwiftDriver/Jobs/LinkJob.swift b/Sources/SwiftDriver/Jobs/LinkJob.swift index 956d4e0bd..fa7d40eb5 100644 --- a/Sources/SwiftDriver/Jobs/LinkJob.swift +++ b/Sources/SwiftDriver/Jobs/LinkJob.swift @@ -15,14 +15,16 @@ import struct TSCBasic.RelativePath extension Driver { internal var relativeOutputFileForImage: RelativePath { - if inputFiles.count == 1 && moduleOutputInfo.nameIsFallback && inputFiles[0].file != .standardInput { - return RelativePath(inputFiles[0].file.basenameWithoutExt) - } + get throws { + if inputFiles.count == 1 && moduleOutputInfo.nameIsFallback && inputFiles[0].file != .standardInput { + return try RelativePath(validating: inputFiles[0].file.basenameWithoutExt) + } - let outputName = + let outputName = toolchain.makeLinkerOutputFilename(moduleName: moduleOutputInfo.name, type: linkerOutputType!) - return RelativePath(outputName) + return try RelativePath(validating: outputName) + } } /// Compute the output file for an image output. @@ -73,6 +75,10 @@ extension Driver { targetInfo: frontendTargetInfo ) + if parsedOptions.hasArgument(.explicitAutoLinking) { + try explicitDependencyBuildPlanner?.getLinkLibraryLoadCommandFlags(&commandLine) + } + return Job( moduleName: moduleOutputInfo.name, kind: .link, diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift index 5fe19de30..718623bab 100644 --- a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift @@ -26,8 +26,8 @@ extension Driver { // Input file list. if shouldUseInputFileList { commandLine.appendFlag(.filelist) - let fileList = VirtualPath.createUniqueFilelist(RelativePath("inputs"), - .list(inputsFromOutputs.map { $0.file })) + let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "inputs"), + .list(inputsFromOutputs.map { $0.file })) commandLine.appendPath(fileList) inputs.append(contentsOf: inputsFromOutputs) @@ -55,27 +55,17 @@ extension Driver { commandLine.appendFlag(.disableSilPerfOptzns) try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .mergeModule, bridgingHeaderHandling: .parsed) - // FIXME: Add MSVC runtime library flags + try addRuntimeLibraryFlags(commandLine: &commandLine) - try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: true) + try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, moduleOutputPaths: moduleOutputPaths, isMergeModule: true) try addCommonSymbolGraphOptions(commandLine: &commandLine) - // Propagate the disable flag for cross-module incremental builds - // if necessary. Note because we're interested in *disabling* this feature, - // we consider the disable form to be the positive and enable to be the - // negative. - if parsedOptions.hasFlag(positive: .disableIncrementalImports, - negative: .enableIncrementalImports, - default: false) { - try commandLine.appendLast(.disableIncrementalImports, from: &parsedOptions) - } - let outputPath = VirtualPath.lookup(moduleOutputInfo.output!.outputPath) commandLine.appendFlag(.o) commandLine.appendPath(outputPath) - if let abiPath = abiDescriptorPath { + if let abiPath = moduleOutputPaths.abiDescriptorFilePath { commandLine.appendFlag(.emitAbiDescriptorPath) commandLine.appendPath(abiPath.file) outputs.append(abiPath) diff --git a/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift b/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift index e41fd4561..9402d26f6 100644 --- a/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift +++ b/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift @@ -18,7 +18,6 @@ extension Driver { // Add the input. commandLine.append(.path(moduleInput.file)) - assert(compilerOutputType == .object, "-modulewrap mode only produces object files") commandLine.appendFlags("-target", targetTriple.triple) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index c52d03410..d224cb54e 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -13,7 +13,6 @@ import SwiftOptions import class Foundation.JSONDecoder -import TSCBasic // <<< import protocol TSCBasic.DiagnosticData import struct TSCBasic.AbsolutePath import struct TSCBasic.Diagnostic @@ -39,54 +38,22 @@ public enum PlanningError: Error, DiagnosticData { } } -/// When emitting bitcode, if the first compile job is scheduled, the second must be. -/// So, group them together for incremental build purposes. -struct CompileJobGroup { - let compileJob: Job - let backendJob: Job? - - init(compileJob: Job, backendJob: Job?) { - assert(compileJob.kind == .compile) - assert(compileJob.primaryInputs.count == 1, "must be unbatched") - assert(backendJob?.kind ?? .backend == .backend) - self.compileJob = compileJob - self.backendJob = backendJob - } - - func allJobs() -> [Job] { - backendJob.map {[compileJob, $0]} ?? [compileJob] - } - - /// Any type of file that is `partOfSwiftCompilation` - var primaryInput: TypedVirtualPath { - compileJob.primaryInputs[0] - } - - var primarySwiftSourceInput: SwiftSourceFile? { - SwiftSourceFile(ifSource: primaryInput) - } - - var outputs: [TypedVirtualPath] { - allJobs().flatMap {$0.outputs} - } -} - @_spi(Testing) public struct JobsInPhases { /// In WMO mode, also includes the multi-compile & its backends, since there are >1 backend jobs let beforeCompiles: [Job] - let compileGroups: [CompileJobGroup] + let compileJobs: [Job] let afterCompiles: [Job] @_spi(Testing) public var allJobs: [Job] { var r = beforeCompiles - compileGroups.forEach { r.append(contentsOf: $0.allJobs()) } + r.append(contentsOf: compileJobs) r.append(contentsOf: afterCompiles) return r } @_spi(Testing) public static var none = JobsInPhases(beforeCompiles: [], - compileGroups: [], - afterCompiles: []) + compileJobs: [], + afterCompiles: []) } // MARK: Standard build planning @@ -94,7 +61,7 @@ extension Driver { /// Plan a standard compilation, which produces jobs for compiling separate /// primary files. private mutating func planStandardCompile() throws - -> ([Job], IncrementalCompilationState?) { + -> ([Job], IncrementalCompilationState?, InterModuleDependencyGraph?) { precondition(compilerMode.isStandardCompilationForPlanning, "compiler mode \(compilerMode) is handled elsewhere") // Determine the initial state for incremental compilation that is required during @@ -104,10 +71,11 @@ extension Driver { try IncrementalCompilationState.computeIncrementalStateForPlanning(driver: &self) // For an explicit build, compute the inter-module dependency graph - let interModuleDependencyGraph = try computeInterModuleDependencyGraph(with: initialIncrementalState) + let interModuleDependencyGraph = try computeInterModuleDependencyGraph() // Compute the set of all jobs required to build this module - let jobsInPhases = try computeJobsForPhasedStandardBuild(with: interModuleDependencyGraph) + let jobsInPhases = try computeJobsForPhasedStandardBuild(moduleDependencyGraph: interModuleDependencyGraph, + initialIncrementalState: initialIncrementalState) // Determine the state for incremental compilation let incrementalCompilationState: IncrementalCompilationState? @@ -117,45 +85,46 @@ extension Driver { try IncrementalCompilationState(driver: &self, jobsInPhases: jobsInPhases, initialState: initialState, - interModuleDependencyGraph: interModuleDependencyGraph) + interModuleDepGraph: interModuleDependencyGraph) } else { incrementalCompilationState = nil } - return try ( - // For compatibility with swiftpm, the driver produces batched jobs - // for every job, even when run in incremental mode, so that all jobs - // can be returned from `planBuild`. - // But in that case, don't emit lifecycle messages. - formBatchedJobs(jobsInPhases.allJobs, - showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil), - incrementalCompilationState - ) + let batchedJobs: [Job] + // If the jobs are batched during the incremental build, reuse the computation rather than computing the batches again. + if let incrementalState = incrementalCompilationState { + // For compatibility reasons, all the jobs planned will be returned, even the incremental state suggests the job is not mandatory. + batchedJobs = incrementalState.skippedJobs + incrementalState.skippedJobsNonCompile + incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles + } else { + batchedJobs = try formBatchedJobs(jobsInPhases.allJobs, + showJobLifecycle: showJobLifecycle, + jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})) + } + + return (batchedJobs, incrementalCompilationState, interModuleDependencyGraph) } /// If performing an explicit module build, compute an inter-module dependency graph. /// If performing an incremental build, and the initial incremental state contains a valid /// graph already, it is safe to re-use without repeating the scan. - private mutating func computeInterModuleDependencyGraph(with initialIncrementalState: - IncrementalCompilationState.InitialStateForPlanning?) + private mutating func computeInterModuleDependencyGraph() throws -> InterModuleDependencyGraph? { - let interModuleDependencyGraph: InterModuleDependencyGraph? - if parsedOptions.contains(.driverExplicitModuleBuild) || - parsedOptions.contains(.explainModuleDependency) { - // If the incremental build record's module dependency graph is up-to-date, we - // can skip dependency scanning entirely. - interModuleDependencyGraph = - try initialIncrementalState?.upToDatePriorInterModuleDependencyGraph ?? - gatherModuleDependencies() + if (parsedOptions.contains(.driverExplicitModuleBuild) || + parsedOptions.contains(.explainModuleDependency)) && + inputFiles.contains(where: { $0.type.isPartOfSwiftCompilation }) { + return try gatherModuleDependencies() } else { - interModuleDependencyGraph = nil + return nil } - return interModuleDependencyGraph } /// Construct a build plan consisting of *all* jobs required for building the current module (non-incrementally). /// At build time, incremental state will be used to distinguish which of these jobs must run. - @_spi(Testing) public mutating func computeJobsForPhasedStandardBuild(with dependencyGraph: InterModuleDependencyGraph?) + /// + /// For Explicitly-Built module dependencies, filter out all up-to-date modules. + @_spi(Testing) public mutating func computeJobsForPhasedStandardBuild(moduleDependencyGraph: InterModuleDependencyGraph?, + initialIncrementalState: + IncrementalCompilationState.InitialStateForPlanning? = nil) throws -> JobsInPhases { // Centralize job accumulation here. // For incremental compilation, must separate jobs happening before, @@ -166,24 +135,25 @@ extension Driver { jobsBeforeCompiles.append(job) } - var compileJobGroups = [CompileJobGroup]() - func addCompileJobGroup(_ group: CompileJobGroup) { - compileJobGroups.append(group) + var compileJobs = [Job]() + func addCompileJob(_ job: Job) { + compileJobs.append(job) } // need to buffer these to dodge shared ownership var jobsAfterCompiles = [Job]() - func addJobAfterCompiles(_ j: Job) { - jobsAfterCompiles.append(j) + func addJobAfterCompiles(_ job: Job) { + jobsAfterCompiles.append(job) } - try addPrecompileModuleDependenciesJobs(dependencyGraph: dependencyGraph, + try addPrecompileModuleDependenciesJobs(dependencyGraph: moduleDependencyGraph, + initialIncrementalState: initialIncrementalState, addJob: addJobBeforeCompiles) try addPrecompileBridgingHeaderJob(addJob: addJobBeforeCompiles) let linkerInputs = try addJobsFeedingLinker( addJobBeforeCompiles: addJobBeforeCompiles, jobsBeforeCompiles: jobsBeforeCompiles, - addCompileJobGroup: addCompileJobGroup, + addCompileJob: addCompileJob, addJobAfterCompiles: addJobAfterCompiles) try addAPIDigesterJobs(addJob: addJobAfterCompiles) try addLinkAndPostLinkJobs(linkerInputs: linkerInputs, @@ -191,12 +161,13 @@ extension Driver { addJob: addJobAfterCompiles) return JobsInPhases(beforeCompiles: jobsBeforeCompiles, - compileGroups: compileJobGroups, + compileJobs: compileJobs, afterCompiles: jobsAfterCompiles) } private mutating func addPrecompileModuleDependenciesJobs( dependencyGraph: InterModuleDependencyGraph?, + initialIncrementalState: IncrementalCompilationState.InitialStateForPlanning?, addJob: (Job) -> Void) throws { guard let resolvedDependencyGraph = dependencyGraph else { @@ -208,7 +179,23 @@ extension Driver { // We may have a dependency graph but not be required to add pre-compile jobs to the build plan, // for example when `-explain-dependency` is being used. guard parsedOptions.contains(.driverExplicitModuleBuild) else { return } - modulePrebuildJobs.forEach(addJob) + + // Filter out the tasks for building module dependencies which are already up-to-date + let mandatoryModuleCompileJobs: [Job] + // If -always-rebuild-module-dependencies is specified, no filtering needed + if parsedOptions.contains(.alwaysRebuildModuleDependencies) { + mandatoryModuleCompileJobs = modulePrebuildJobs + } else { + let enableIncrementalRemarks = initialIncrementalState != nil && initialIncrementalState!.incrementalOptions.contains(.showIncremental) + let reporter: IncrementalCompilationState.Reporter? = enableIncrementalRemarks ? + IncrementalCompilationState.Reporter(diagnosticEngine: diagnosticEngine, outputFileMap: outputFileMap) : nil + mandatoryModuleCompileJobs = + try resolvedDependencyGraph.filterMandatoryModuleDependencyCompileJobs(modulePrebuildJobs, + fileSystem: fileSystem, + cas: cas, + reporter: reporter) + } + mandatoryModuleCompileJobs.forEach(addJob) } @@ -226,19 +213,27 @@ extension Driver { ) } - private mutating func addEmitModuleJob(addJobBeforeCompiles: (Job) -> Void, pchCompileJob: Job?) throws -> Job? { - if emitModuleSeparately { - let emitJob = try emitModuleJob(pchCompileJob: pchCompileJob) - addJobBeforeCompiles(emitJob) - return emitJob - } - return nil + private mutating func addEmitModuleJob( + addJobBeforeCompiles: (Job) -> Void, + pchCompileJob: Job?, + isVariantModule: Bool = false) throws -> Job? { + // The target variant module is always emitted separately, so we need to + // add an explicit job regardless of whether the primary target was + // emitted separately + if emitModuleSeparately || isVariantModule { + let emitJob = try emitModuleJob( + pchCompileJob: pchCompileJob, + isVariantJob: isVariantModule) + addJobBeforeCompiles(emitJob) + return emitJob + } + return nil } private mutating func addJobsFeedingLinker( addJobBeforeCompiles: (Job) -> Void, jobsBeforeCompiles: [Job], - addCompileJobGroup: (CompileJobGroup) -> Void, + addCompileJob: (Job) -> Void, addJobAfterCompiles: (Job) -> Void ) throws -> [TypedVirtualPath] { @@ -307,8 +302,15 @@ extension Driver { addLinkerInput: addLinkerInput) } + if variantModuleOutputInfo != nil { + _ = try addEmitModuleJob( + addJobBeforeCompiles: addJobBeforeCompiles, + pchCompileJob: jobCreatingPch, + isVariantModule: true) + } + try addJobsForPrimaryInputs( - addCompileJobGroup: addCompileJobGroup, + addCompileJob: addCompileJob, addModuleInput: addModuleInput, addLinkerInput: addLinkerInput, addJobOutputs: addJobOutputs, @@ -345,37 +347,20 @@ extension Driver { guard case .singleCompile = compilerMode, inputFiles.contains(where: { $0.type.isPartOfSwiftCompilation }) else { return nil } - - if parsedOptions.hasArgument(.embedBitcode), - inputFiles.allSatisfy({ $0.type.isPartOfSwiftCompilation }) { - let compile = try compileJob(primaryInputs: [], - outputType: .llvmBitcode, - addJobOutputs: addJobOutputs, - pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) - addJob(compile) - let backendJobs = try compile.outputs.compactMap { output in - output.type == .llvmBitcode - ? try backendJob(input: output, baseInput: nil, addJobOutputs: addJobOutputs) - : nil - } - backendJobs.forEach(addJob) - return compile - } else { - // We can skip the compile jobs if all we want is a module when it's - // built separately. - let compile = try compileJob(primaryInputs: [], - outputType: compilerOutputType, - addJobOutputs: addJobOutputs, - pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) - addJob(compile) - return compile - } + // We can skip the compile jobs if all we want is a module when it's + // built separately. + let compile = try compileJob(primaryInputs: [], + outputType: compilerOutputType, + addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, + emitModuleTrace: emitModuleTrace, + produceCacheKey: true) + addJob(compile) + return compile } private mutating func addJobsForPrimaryInputs( - addCompileJobGroup: (CompileJobGroup) -> Void, + addCompileJob: (Job) -> Void, addModuleInput: (TypedVirtualPath) -> Void, addLinkerInput: (TypedVirtualPath) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void, @@ -388,7 +373,7 @@ extension Driver { // Only emit a loaded module trace from the first frontend job. try addJobForPrimaryInput( input: input, - addCompileJobGroup: addCompileJobGroup, + addCompileJob: addCompileJob, addModuleInput: addModuleInput, addLinkerInput: addLinkerInput, addJobOutputs: addJobOutputs, @@ -399,7 +384,7 @@ extension Driver { private mutating func addJobForPrimaryInput( input: TypedVirtualPath, - addCompileJobGroup: (CompileJobGroup) -> Void, + addCompileJob: (Job) -> Void, addModuleInput: (TypedVirtualPath) -> Void, addLinkerInput: (TypedVirtualPath) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void, @@ -416,11 +401,11 @@ extension Driver { // We can skip the compile jobs if all we want is a module when it's // built separately. let canSkipIfOnlyModule = compilerOutputType == .swiftModule && emitModuleSeparately - try createAndAddCompileJobGroup(primaryInput: input, + try createAndAddCompileJob(primaryInput: input, emitModuleTrace: emitModuleTrace, canSkipIfOnlyModule: canSkipIfOnlyModule, pchCompileJob: pchCompileJob, - addCompileJobGroup: addCompileJobGroup, + addCompileJob: addCompileJob, addJobOutputs: addJobOutputs) case .object, .autolink, .llvmBitcode, .tbd: @@ -449,42 +434,26 @@ extension Driver { } } - private mutating func createAndAddCompileJobGroup( + private mutating func createAndAddCompileJob( primaryInput: TypedVirtualPath, emitModuleTrace: Bool, canSkipIfOnlyModule: Bool, pchCompileJob: Job?, - addCompileJobGroup: (CompileJobGroup) -> Void, + addCompileJob: (Job) -> Void, addJobOutputs: ([TypedVirtualPath]) -> Void ) throws { - if parsedOptions.hasArgument(.embedBitcode), - inputFiles.allSatisfy({ $0.type.isPartOfSwiftCompilation }) { - let compile = try compileJob(primaryInputs: [primaryInput], - outputType: .llvmBitcode, - addJobOutputs: addJobOutputs, - pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) - let backendJobs = try compile.outputs.compactMap { output in - output.type == .llvmBitcode - ? try backendJob(input: output, baseInput: primaryInput, addJobOutputs: addJobOutputs) - : nil - } - assert(backendJobs.count <= 1) - addCompileJobGroup(CompileJobGroup(compileJob: compile, backendJob: backendJobs.first)) - } else { - // TODO: if !canSkipIfOnlyModule { - // Some other tools still expect the partial jobs. Bring this check - // back once they are updated. rdar://84979778 - - // We can skip the compile jobs if all we want is a module when it's - // built separately. - let compile = try compileJob(primaryInputs: [primaryInput], - outputType: compilerOutputType, - addJobOutputs: addJobOutputs, - pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) - addCompileJobGroup(CompileJobGroup(compileJob: compile, backendJob: nil)) - } + // We can skip the compile jobs if all we want is a module when it's + // built separately. + if parsedOptions.hasArgument(.driverExplicitModuleBuild), canSkipIfOnlyModule { return } + // If we are in the batch mode, the constructed jobs here will be batched + // later. There is no need to produce cache key for the job. + let compile = try compileJob(primaryInputs: [primaryInput], + outputType: compilerOutputType, + addJobOutputs: addJobOutputs, + pchCompileJob: pchCompileJob, + emitModuleTrace: emitModuleTrace, + produceCacheKey: !compilerMode.isBatchCompile) + addCompileJob(compile) } /// Need a merge module job if there are module inputs @@ -549,10 +518,6 @@ extension Driver { private mutating func addVerifyJobs(emitModuleJob: Job, addJob: (Job) -> Void ) throws { - // Turn this flag on by default with the env var or for public frameworks. - let onByDefault = env["ENABLE_DEFAULT_INTERFACE_VERIFIER"] != nil || - parsedOptions.getLastArgument(.libraryLevel)?.asSingle == "api" - guard // Only verify modules with library evolution. parsedOptions.hasArgument(.enableLibraryEvolution), @@ -560,7 +525,7 @@ extension Driver { // Only verify when requested, on by default and not disabled. parsedOptions.hasFlag(positive: .verifyEmittedModuleInterface, negative: .noVerifyEmittedModuleInterface, - default: onByDefault), + default: true), // Don't verify by default modules emitted from a merge-module job // as it's more likely to be invalid. @@ -570,27 +535,61 @@ extension Driver { default: false) else { return } - let optIn = env["ENABLE_DEFAULT_INTERFACE_VERIFIER"] != nil || - parsedOptions.hasArgument(.verifyEmittedModuleInterface) - func addVerifyJob(forPrivate: Bool) throws { - let isNeeded = - forPrivate - ? parsedOptions.hasArgument(.emitPrivateModuleInterfacePath) - : parsedOptions.hasArgument(.emitModuleInterface, .emitModuleInterfacePath) + // Downgrade errors to a warning for modules expected to fail this check. + var knownFailingModules: Set = ["TestBlocklistedModule"] + knownFailingModules = knownFailingModules.union( + Driver.getAllConfiguredModules(withKey: "SkipModuleInterfaceVerify", + getAdopterConfigsFromXcodeDefaultToolchain())) + + let moduleName = parsedOptions.getLastArgument(.moduleName)?.asSingle + let reportAsError = !knownFailingModules.contains(moduleName ?? "") || + env["ENABLE_DEFAULT_INTERFACE_VERIFIER"] != nil || + parsedOptions.hasFlag(positive: .verifyEmittedModuleInterface, + negative: .noVerifyEmittedModuleInterface, + default: false) + + if !reportAsError { + diagnosticEngine + .emit( + .remark( + "Verification of module interfaces for '\(moduleName ?? "No module name")' set to warning only by blocklist")) + } + + enum InterfaceMode { + case Public, Private, Package + } + + func addVerifyJob(for mode: InterfaceMode) throws { + var isNeeded = false + var outputType: FileType + + switch mode { + case .Public: + isNeeded = parsedOptions.hasArgument(.emitModuleInterface, .emitModuleInterfacePath) + outputType = FileType.swiftInterface + case .Private: + isNeeded = parsedOptions.hasArgument(.emitPrivateModuleInterfacePath) + outputType = .privateSwiftInterface + case .Package: + isNeeded = parsedOptions.hasArgument(.emitPackageModuleInterfacePath) + outputType = .packageSwiftInterface + } + guard isNeeded else { return } - let outputType: FileType = - forPrivate ? .privateSwiftInterface : .swiftInterface let mergeInterfaceOutputs = emitModuleJob.outputs.filter { $0.type == outputType } assert(mergeInterfaceOutputs.count == 1, "Merge module job should only have one swiftinterface output") let job = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0], emitModuleJob: emitModuleJob, - optIn: optIn) + reportAsError: reportAsError) addJob(job) } - try addVerifyJob(forPrivate: false) - try addVerifyJob(forPrivate: true) + try addVerifyJob(for: .Public) + try addVerifyJob(for: .Private) + if parsedOptions.hasArgument(.packageName) { + try addVerifyJob(for: .Package) + } } private mutating func addAutolinkExtractJob( @@ -617,21 +616,20 @@ extension Driver { addJob: (Job) -> Void, addLinkerInput: (TypedVirtualPath) -> Void) throws { - guard case .astTypes = debugInfo.level - else { return } - if targetTriple.objectFormat != .macho { + guard case .astTypes = debugInfo.level else { return } + + let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } + assert(mergeModuleOutputs.count == 1, + "Merge module job should only have one swiftmodule output") + + if targetTriple.objectFormat == .macho { + addLinkerInput(mergeModuleOutputs[0]) + } else { // Module wrapping is required. - let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } - assert(mergeModuleOutputs.count == 1, - "Merge module job should only have one swiftmodule output") let wrapJob = try moduleWrapJob(moduleInput: mergeModuleOutputs[0]) addJob(wrapJob) + wrapJob.outputs.forEach(addLinkerInput) - } else { - let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } - assert(mergeModuleOutputs.count == 1, - "Merge module job should only have one swiftmodule output") - addLinkerInput(mergeModuleOutputs[0]) } } @@ -689,8 +687,9 @@ extension Driver { dependencyOracle: interModuleDependencyOracle, integratedDriver: integratedDriver, supportsExplicitInterfaceBuild: - isFrontendArgSupported(.explicitInterfaceModuleBuild), - enableCAS: enableCaching) + isFrontendArgSupported(.explicitInterfaceModuleBuild), + cas: cas, + prefixMap: prefixMapping) return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() } @@ -714,6 +713,15 @@ extension Driver { swiftCompilerPrefixArgs: swiftCompilerPrefixArgs) } + if parsedOptions.hasArgument(.printSupportedFeatures) { + try verifyFrontendSupportsOptionIfNecessary(.printSupportedFeatures) + + return try toolchain.printSupportedFeaturesJob( + requiresInPlaceExecution: true, + swiftCompilerPrefixArgs: swiftCompilerPrefixArgs + ) + } + if parsedOptions.hasArgument(.version) || parsedOptions.hasArgument(.version_) { return Job( moduleName: moduleOutputInfo.name, @@ -748,17 +756,17 @@ extension Driver { /// Plan a build by producing a set of jobs to complete the build. /// Should be private, but compiler bug /*private*/ mutating func planPossiblyIncrementalBuild() throws - -> ([Job], IncrementalCompilationState?) { + -> ([Job], IncrementalCompilationState?, InterModuleDependencyGraph?) { if let job = try immediateForwardingJob() { - return ([job], nil) + return ([job], nil, nil) } // The REPL doesn't require input files, but all other modes do. guard !inputFiles.isEmpty || compilerMode == .repl || compilerMode == .intro else { if parsedOptions.hasArgument(.v) { // `swiftc -v` is allowed and prints version information. - return ([], nil) + return ([], nil, nil) } throw Error.noInputFiles } @@ -769,7 +777,7 @@ extension Driver { if !inputFiles.isEmpty { throw PlanningError.replReceivedInput } - return ([try replJob()], nil) + return ([try replJob()], nil, nil) case .immediate: var jobs: [Job] = [] @@ -778,9 +786,10 @@ extension Driver { try parsedOptions.contains(.driverExplicitModuleBuild) ? gatherModuleDependencies() : nil try addPrecompileModuleDependenciesJobs(dependencyGraph: moduleDependencyGraph, + initialIncrementalState: nil, addJob: { jobs.append($0) }) jobs.append(try interpretJob(inputs: inputFiles)) - return (jobs, nil) + return (jobs, nil, nil) case .standardCompile, .batchCompile, .singleCompile: return try planStandardCompile() @@ -789,15 +798,15 @@ extension Driver { if inputFiles.count != 1 { throw PlanningError.emitPCMWrongInputFiles } - return ([try generateEmitPCMJob(input: inputFiles.first!)], nil) + return ([try generateEmitPCMJob(input: inputFiles.first!)], nil, nil) case .dumpPCM: if inputFiles.count != 1 { throw PlanningError.dumpPCMWrongInputFiles } - return ([try generateDumpPCMJob(input: inputFiles.first!)], nil) + return ([try generateDumpPCMJob(input: inputFiles.first!)], nil, nil) case .intro: - return (try helpIntroJobs(), nil) + return (try helpIntroJobs(), nil, nil) } } } @@ -821,7 +830,7 @@ extension Driver { /// /// So, in order to avoid making jobs and rebatching, the code would have to just get outputs for each /// compilation. But `compileJob` intermixes the output computation with other stuff. - mutating func formBatchedJobs(_ jobs: [Job], showJobLifecycle: Bool) throws -> [Job] { + mutating func formBatchedJobs(_ jobs: [Job], showJobLifecycle: Bool, jobCreatingPch: Job?) throws -> [Job] { guard compilerMode.isBatchCompile else { // Don't even go through the logic so as to not print out confusing // "batched foobar" messages. @@ -839,15 +848,11 @@ extension Driver { let partitions = batchPartitions( inputs: inputsInOrder, showJobLifecycle: showJobLifecycle) - let outputType = parsedOptions.hasArgument(.embedBitcode) - ? .llvmBitcode - : compilerOutputType let inputsRequiringModuleTrace = Set( compileJobs.filter { $0.outputs.contains {$0.type == .moduleTrace} } .flatMap {$0.primaryInputs} ) - let jobCreatingPch: Job? = jobs.first(where: {$0.kind == .generatePCH}) let batchedCompileJobs = try inputsInOrder.compactMap { anInput -> Job? in let idx = partitions.assignment[anInput]! @@ -876,10 +881,11 @@ extension Driver { let constituentsEmittedModuleTrace = !inputsRequiringModuleTrace.intersection(primaryInputs).isEmpty // no need to add job outputs again return try compileJob(primaryInputs: primaryInputs, - outputType: outputType, + outputType: compilerOutputType, addJobOutputs: {_ in }, pchCompileJob: jobCreatingPch, - emitModuleTrace: constituentsEmittedModuleTrace) + emitModuleTrace: constituentsEmittedModuleTrace, + produceCacheKey: true) } return batchedCompileJobs + noncompileJobs } diff --git a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift index df08f08a2..e8a557670 100644 --- a/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift +++ b/Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift @@ -13,7 +13,6 @@ import SwiftOptions import class Foundation.JSONEncoder import class Foundation.JSONSerialization -import TSCBasic // <<< import class TSCBasic.DiagnosticsEngine import protocol TSCBasic.WritableByteStream import struct TSCBasic.AbsolutePath @@ -127,7 +126,7 @@ fileprivate func logOutput(_ job: Job, _ result: ProcessResult, _ logPath: Absol } let fileName = "\(job.moduleName)-\(interfaceBase)-\(stdout ? "out" : errKind.rawValue).txt" try localFileSystem.writeFileContents(logPath.appending(component: fileName)) { - $0 <<< content + $0.send(content) } } @@ -136,12 +135,36 @@ func printJobInfo(_ job: Job, _ start: Bool, _ verbose: Bool) { return } Driver.stdErrQueue.sync { - stderrStream <<< (start ? "started: " : "finished: ") - stderrStream <<< getLastInputPath(job).pathString <<< "\n" + stderrStream.send(start ? "started: " : "finished: ") + stderrStream.send("\(getLastInputPath(job).pathString)\n") stderrStream.flush() } } +class JSONOutputDelegate: Encodable { + enum SDKFailureKind: String, Encodable { + case BrokenTextualInterface + } + struct SDKFailure: Encodable { + let inputPath: String + let kind: SDKFailureKind + } + var allFailures = [SDKFailure]() + func jobFinished(_ job: Job, _ result: ProcessResult) { + switch result.exitStatus { + case .terminated(code: let code): + if code == 0 { + break + } else { + allFailures.append(SDKFailure(inputPath: getLastInputPath(job).pathString, + kind: .BrokenTextualInterface)) + } + default: + break + } + } +} + fileprivate class ModuleCompileDelegate: JobExecutionDelegate { var failingModules = Set() var commandMap: [Int: String] = [:] @@ -149,11 +172,15 @@ fileprivate class ModuleCompileDelegate: JobExecutionDelegate { let verbose: Bool var failingCriticalOutputs: Set let logPath: AbsolutePath? - public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool, _ logPath: AbsolutePath?) { + let jsonDelegate: JSONOutputDelegate + var compiledModules: [String: Int] = [:] + init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool, + _ logPath: AbsolutePath?, _ jsonDelegate: JSONOutputDelegate) { self.diagnosticsEngine = diagnosticsEngine self.verbose = verbose self.failingCriticalOutputs = Set(jobs.compactMap(ModuleCompileDelegate.getCriticalOutput)) self.logPath = logPath + self.jsonDelegate = jsonDelegate } /// Dangling jobs are macabi-only modules. We should run those jobs if foundation @@ -175,18 +202,46 @@ fileprivate class ModuleCompileDelegate: JobExecutionDelegate { return !failingCriticalOutputs.isEmpty } + public func checkCriticalModulesGenerated() -> Bool { + let sortedModules = compiledModules.sorted(by: <) + Driver.stdErrQueue.sync { + stderrStream.send("===================================================\n") + sortedModules.forEach { + stderrStream.send("\($0.key): \($0.value)\n") + } + stderrStream.send("===================================================\n") + stderrStream.flush() + } + let keyModules = ["Swift", "SwiftUI", "Foundation"] + return keyModules.allSatisfy { + if compiledModules.keys.contains($0) { + return true + } + stderrStream.send("Missing critical module: \($0)\n") + return false + } + } + public func jobFinished(job: Job, result: ProcessResult, pid: Int) { + self.jsonDelegate.jobFinished(job, result) switch result.exitStatus { case .terminated(code: let code): if code == 0 { printJobInfo(job, false, verbose) failingCriticalOutputs.remove(job.outputs[0].file) + + // Keep track of Swift modules that have been already generated. + if let seen = compiledModules[job.moduleName] { + compiledModules[job.moduleName] = seen + 1 + } else { + compiledModules[job.moduleName] = 1 + } } else { failingModules.insert(job.moduleName) let result: String = try! result.utf8stderrOutput() Driver.stdErrQueue.sync { - stderrStream <<< "failed: " <<< commandMap[pid]! <<< "\n" - stderrStream <<< result <<< "\n" + stderrStream.send("failed: \(commandMap[pid]!)\n") + stderrStream.send("\(result)\n") stderrStream.flush() } } @@ -203,7 +258,7 @@ fileprivate class ModuleCompileDelegate: JobExecutionDelegate { try logOutput(job, result, logPath, false) } catch { Driver.stdErrQueue.sync { - stderrStream <<< "Failed to generate log file" + stderrStream.send("Failed to generate log file") stderrStream.flush() } } @@ -241,7 +296,7 @@ fileprivate class ABICheckingDelegate: JobExecutionDelegate { try logOutput(job, result, logPath, false) } catch { Driver.stdErrQueue.sync { - stderrStream <<< "Failed to generate log file" + stderrStream.send("Failed to generate log file") stderrStream.flush() } } @@ -250,6 +305,7 @@ fileprivate class ABICheckingDelegate: JobExecutionDelegate { public class PrebuiltModuleGenerationDelegate: JobExecutionDelegate { + fileprivate let jsonDelegate: JSONOutputDelegate fileprivate let compileDelegate: ModuleCompileDelegate fileprivate let abiCheckDelegate: ABICheckingDelegate @@ -265,7 +321,10 @@ public class PrebuiltModuleGenerationDelegate: JobExecutionDelegate { public init(_ jobs: [Job], _ diagnosticsEngine: DiagnosticsEngine, _ verbose: Bool, _ logPath: AbsolutePath?) { - self.compileDelegate = ModuleCompileDelegate(jobs.filter(ModuleCompileDelegate.canHandle), diagnosticsEngine, verbose, logPath) + self.jsonDelegate = JSONOutputDelegate() + self.compileDelegate = ModuleCompileDelegate(jobs.filter(ModuleCompileDelegate.canHandle), + diagnosticsEngine, verbose, logPath, + self.jsonDelegate) self.abiCheckDelegate = ABICheckingDelegate(verbose, logPath) } @@ -286,6 +345,16 @@ public class PrebuiltModuleGenerationDelegate: JobExecutionDelegate { public var hasCriticalFailure: Bool { return compileDelegate.hasCriticalFailure } + public func checkCriticalModulesGenerated() -> Bool { + return compileDelegate.checkCriticalModulesGenerated() + } + public func emitJsonOutput(to path: AbsolutePath) throws { + let data = try JSONEncoder().encode(self.jsonDelegate) + if let json = try? JSONSerialization.jsonObject(with: data, options: .mutableContainers), + let jsonData = try? JSONSerialization.data(withJSONObject: json, options: .prettyPrinted) { + try localFileSystem.writeFileContents(path, bytes: ByteString(jsonData)) + } + } } public struct PrebuiltModuleInput { @@ -315,6 +384,7 @@ public class SwiftAdopter: Codable { public let moduleDir: String public let hasInterface: Bool public let hasPrivateInterface: Bool + public let hasPackageInterface: Bool public let hasModule: Bool public let isFramework: Bool public let isPrivate: Bool @@ -326,6 +396,7 @@ public class SwiftAdopter: Codable { self.moduleDir = SwiftAdopter.relativeToSDK(moduleDir) self.hasInterface = !hasInterface.isEmpty self.hasPrivateInterface = hasInterface.contains { $0.basename.hasSuffix(".private.swiftinterface") } + self.hasPackageInterface = hasInterface.contains { $0.basename.hasSuffix(".package.swiftinterface") } self.hasModule = !hasModule.isEmpty self.isFramework = self.moduleDir.contains("\(name).framework") self.isPrivate = self.moduleDir.contains("PrivateFrameworks") @@ -393,12 +464,21 @@ typealias PrebuiltModuleOutput = PrebuiltModuleInput public struct SDKPrebuiltModuleInputsCollector { let sdkPath: AbsolutePath - let nonFrameworkDirs = [RelativePath("usr/lib/swift"), - RelativePath("System/iOSSupport/usr/lib/swift")] - let frameworkDirs = [RelativePath("System/Library/Frameworks"), - RelativePath("System/Library/PrivateFrameworks"), - RelativePath("System/iOSSupport/System/Library/Frameworks"), - RelativePath("System/iOSSupport/System/Library/PrivateFrameworks")] + var nonFrameworkDirs: [RelativePath] { + get throws { + try [RelativePath(validating: "usr/lib/swift"), + RelativePath(validating: "System/iOSSupport/usr/lib/swift")] + } + } + var frameworkDirs: [RelativePath] { + get throws { + try [RelativePath(validating: "System/Library/Frameworks"), + RelativePath(validating: "System/Library/PrivateFrameworks"), + RelativePath(validating: "System/iOSSupport/System/Library/Frameworks"), + RelativePath(validating: "System/iOSSupport/System/Library/PrivateFrameworks")] + } + } + let sdkInfo: DarwinToolchain.DarwinSDKInfo let diagEngine: DiagnosticsEngine public init(_ sdkPath: AbsolutePath, _ diagEngine: DiagnosticsEngine) { @@ -430,6 +510,10 @@ public struct SDKPrebuiltModuleInputsCollector { return "arm64-apple-tvos\(version)" case .appletvsimulator: return "arm64-apple-tvos\(version)-simulator" + case .visionos: + return "arm64-apple-xros\(version)" + case .visionsimulator: + return "arm64-apple-xros\(version)-simulator" case .unknown: fatalError("unknown platform kind") } @@ -483,7 +567,7 @@ public struct SDKPrebuiltModuleInputsCollector { allSwiftAdopters.append(try! SwiftAdopter(moduleName, dir, hasInterface, hasModule)) } // Search inside framework dirs in an SDK to find .swiftmodule directories. - for dir in frameworkDirs { + for dir in try frameworkDirs { let frameDir = AbsolutePath(sdkPath, dir) if !localFileSystem.exists(frameDir) { continue @@ -501,14 +585,14 @@ public struct SDKPrebuiltModuleInputsCollector { } } // Search inside lib dirs in an SDK to find .swiftmodule directories. - for dir in nonFrameworkDirs { + for dir in try nonFrameworkDirs { let swiftModuleDir = AbsolutePath(sdkPath, dir) if !localFileSystem.exists(swiftModuleDir) { continue } try localFileSystem.getDirectoryContents(swiftModuleDir).forEach { if $0.hasSuffix(".swiftmodule") { - try updateResults(AbsolutePath(swiftModuleDir, $0)) + try updateResults(try AbsolutePath(validating: $0, relativeTo: swiftModuleDir)) } } } @@ -529,15 +613,15 @@ extension InterModuleDependencyGraph { func dumpModuleName(_ stream: WritableByteStream, _ dep: ModuleDependencyId) { switch dep { case .swift(let name): - stream <<< "\"\(name).swiftmodule\"" + stream.send("\"\(name).swiftmodule\"") case .clang(let name): - stream <<< "\"\(name).pcm\"" + stream.send("\"\(name).pcm\"") default: break } } - try localFileSystem.writeFileContents(path) {Stream in - Stream <<< "digraph {\n" + try localFileSystem.writeFileContents(path) { Stream in + Stream.send("digraph {\n") for key in modules.keys { switch key { case .swift(let name): @@ -549,20 +633,20 @@ extension InterModuleDependencyGraph { if !includingPCM && isPCM(key) { break } - modules[key]!.directDependencies?.forEach { dep in + modules[key]!.allDependencies.forEach { dep in if !includingPCM && isPCM(dep) { return } dumpModuleName(Stream, key) - Stream <<< " -> " + Stream.send(" -> ") dumpModuleName(Stream, dep) - Stream <<< ";\n" + Stream.send(";\n") } default: break } } - Stream <<< "}\n" + Stream.send("}\n") } } } @@ -730,7 +814,8 @@ extension Driver { func getSwiftDependencies(for module: String) -> [String] { let info = dependencyGraph.modules[.swift(module)]! - guard let dependencies = info.directDependencies else { + let dependencies = info.allDependencies + guard !dependencies.isEmpty else { return [] } return collectSwiftModuleNames(dependencies) @@ -775,31 +860,29 @@ extension Driver { } // Keep track of modules we haven't handled. var unhandledModules = Set(inputMap.keys) - if let importedModules = dependencyGraph.mainModule.directDependencies { - // Start from those modules explicitly imported into the file under scanning - var openModules = collectSwiftModuleNames(importedModules) - var idx = 0 - while idx != openModules.count { - let module = openModules[idx] - let dependencies = getSwiftDependencies(for: module) - try forEachInputOutputPair(module) { input, output in - jobs.append(contentsOf: try generateSingleModuleBuildingJob(module, - prebuiltModuleDir, input, output, - try getOutputPaths(withName: dependencies, loadableFor: input.arch), - currentABIDir, baselineABIDir)) - } - // For each dependency, add to the list to handle if the list doesn't - // contain this dependency. - dependencies.forEach({ newModule in - if !openModules.contains(newModule) { - diagnosticEngine.emit(.note("\(newModule) is discovered."), - location: nil) - openModules.append(newModule) - } - }) - unhandledModules.remove(module) - idx += 1 + // Start from those modules explicitly imported into the file under scanning + var openModules = collectSwiftModuleNames(dependencyGraph.mainModule.allDependencies) + var idx = 0 + while idx != openModules.count { + let module = openModules[idx] + let dependencies = getSwiftDependencies(for: module) + try forEachInputOutputPair(module) { input, output in + jobs.append(contentsOf: try generateSingleModuleBuildingJob(module, + prebuiltModuleDir, input, output, + try getOutputPaths(withName: dependencies, loadableFor: input.arch), + currentABIDir, baselineABIDir)) } + // For each dependency, add to the list to handle if the list doesn't + // contain this dependency. + dependencies.forEach({ newModule in + if !openModules.contains(newModule) { + diagnosticEngine.emit(.note("\(newModule) is discovered."), + location: nil) + openModules.append(newModule) + } + }) + unhandledModules.remove(module) + idx += 1 } // We are done if we don't need to handle all inputs exhaustively. diff --git a/Sources/SwiftDriver/Jobs/PrintSupportedFeaturesJob.swift b/Sources/SwiftDriver/Jobs/PrintSupportedFeaturesJob.swift new file mode 100644 index 000000000..0c7993e33 --- /dev/null +++ b/Sources/SwiftDriver/Jobs/PrintSupportedFeaturesJob.swift @@ -0,0 +1,36 @@ +//===------- PrintSupportedFeaturesJob.swift - Swift Target Info Job ------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension Toolchain { + @_spi(Testing) public func printSupportedFeaturesJob( + requiresInPlaceExecution: Bool = false, + swiftCompilerPrefixArgs: [String] + ) throws -> Job { + var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } + commandLine.append(contentsOf: [ + .flag("-frontend"), + .flag("-print-supported-features"), + ]) + + return Job( + moduleName: "", + kind: .printSupportedFeatures, + tool: try resolvedTool(.swiftCompiler), + commandLine: commandLine, + displayInputs: [], + inputs: [], + primaryInputs: [], + outputs: [.init(file: .standardOutput, type: .jsonSupportedFeatures)], + requiresInPlaceExecution: requiresInPlaceExecution + ) + } +} diff --git a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift index 066eaad95..f4bfb2d15 100644 --- a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift +++ b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift @@ -101,6 +101,8 @@ public struct FrontendTargetInfo: Codable { /// Whether the Swift libraries need to be referenced in their system /// location (/usr/lib/swift) via rpath. let librariesRequireRPath: Bool + + let openbsdBTCFIEnabled: Bool? } @_spi(Testing) public struct Paths: Codable { @@ -181,46 +183,39 @@ extension Toolchain { } extension Driver { - @_spi(Testing) public static func queryTargetInfoInProcess(of toolchain: Toolchain, + @_spi(Testing) public static func queryTargetInfoInProcess(libSwiftScanInstance: SwiftScan, + toolchain: Toolchain, fileSystem: FileSystem, workingDirectory: AbsolutePath?, - invocationCommand: [String]) throws -> FrontendTargetInfo? { - let optionalSwiftScanLibPath = try toolchain.lookupSwiftScanLib() - if let swiftScanLibPath = optionalSwiftScanLibPath, - fileSystem.exists(swiftScanLibPath) { - let libSwiftScanInstance = try SwiftScan(dylib: swiftScanLibPath) - if libSwiftScanInstance.canQueryTargetInfo() { - let cwd = try workingDirectory ?? fileSystem.currentWorkingDirectory ?? fileSystem.tempDirectory - let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path - let targetInfoData = - try libSwiftScanInstance.queryTargetInfoJSON(workingDirectory: cwd, - compilerExecutablePath: compilerExecutablePath, - invocationCommand: invocationCommand) - do { - return try JSONDecoder().decode(FrontendTargetInfo.self, from: targetInfoData) - } catch let decodingError as DecodingError { - let stringToDecode = String(data: targetInfoData, encoding: .utf8) - let errorDesc: String - switch decodingError { - case let .typeMismatch(type, context): - errorDesc = "type mismatch: \(type), path: \(context.codingPath)" - case let .valueNotFound(type, context): - errorDesc = "value missing: \(type), path: \(context.codingPath)" - case let .keyNotFound(key, context): - errorDesc = "key missing: \(key), path: \(context.codingPath)" - case let .dataCorrupted(context): - errorDesc = "data corrupted at path: \(context.codingPath)" - @unknown default: - errorDesc = "unknown decoding error" - } - throw Error.unableToDecodeFrontendTargetInfo( - stringToDecode, - invocationCommand, - errorDesc) - } + invocationCommand: [String]) throws -> FrontendTargetInfo { + let cwd = try workingDirectory ?? fileSystem.currentWorkingDirectory ?? fileSystem.tempDirectory + let compilerExecutablePath = try toolchain.resolvedTool(.swiftCompiler).path + let targetInfoData = + try libSwiftScanInstance.queryTargetInfoJSON(workingDirectory: cwd, + compilerExecutablePath: compilerExecutablePath, + invocationCommand: invocationCommand) + do { + return try JSONDecoder().decode(FrontendTargetInfo.self, from: targetInfoData) + } catch let decodingError as DecodingError { + let stringToDecode = String(data: targetInfoData, encoding: .utf8) + let errorDesc: String + switch decodingError { + case let .typeMismatch(type, context): + errorDesc = "type mismatch: \(type), path: \(context.codingPath)" + case let .valueNotFound(type, context): + errorDesc = "value missing: \(type), path: \(context.codingPath)" + case let .keyNotFound(key, context): + errorDesc = "key missing: \(key), path: \(context.codingPath)" + case let .dataCorrupted(context): + errorDesc = "data corrupted at path: \(context.codingPath)" + @unknown default: + errorDesc = "unknown decoding error" } + throw Error.unableToDecodeFrontendTargetInfo( + stringToDecode, + invocationCommand, + errorDesc) } - return nil } static func computeTargetInfo(target: Triple?, @@ -231,6 +226,7 @@ extension Driver { requiresInPlaceExecution: Bool = false, useStaticResourceDir: Bool = false, swiftCompilerPrefixArgs: [String], + libSwiftScan: SwiftScan?, toolchain: Toolchain, fileSystem: FileSystem, workingDirectory: AbsolutePath?, @@ -243,20 +239,19 @@ extension Driver { requiresInPlaceExecution: requiresInPlaceExecution, useStaticResourceDir: useStaticResourceDir, swiftCompilerPrefixArgs: swiftCompilerPrefixArgs) - var command = try Self.itemizedJobCommand(of: frontendTargetInfoJob, - useResponseFiles: .disabled, - using: executor.resolver) - Self.sanitizeCommandForLibScanInvocation(&command) - - do { - if let targetInfo = - try Self.queryTargetInfoInProcess(of: toolchain, fileSystem: fileSystem, - workingDirectory: workingDirectory, - invocationCommand: command) { - return targetInfo + if let libSwiftScanInstance = libSwiftScan, + libSwiftScanInstance.canQueryTargetInfo() { + do { + var command = try Self.itemizedJobCommand(of: frontendTargetInfoJob, + useResponseFiles: .disabled, + using: executor.resolver) + Self.sanitizeCommandForLibScanInvocation(&command) + return try Self.queryTargetInfoInProcess(libSwiftScanInstance: libSwiftScanInstance, toolchain: toolchain, + fileSystem: fileSystem, workingDirectory: workingDirectory, + invocationCommand: command) + } catch { + diagnosticsEngine.emit(.remark_inprocess_target_info_query_failed(error.localizedDescription)) } - } catch { - diagnosticsEngine.emit(.warning_inprocess_target_info_query_failed(error.localizedDescription)) } // Fallback: Invoke `swift-frontend -print-target-info` and decode the output diff --git a/Sources/SwiftDriver/Jobs/ReplJob.swift b/Sources/SwiftDriver/Jobs/ReplJob.swift index d8bff16e6..e7ff79996 100644 --- a/Sources/SwiftDriver/Jobs/ReplJob.swift +++ b/Sources/SwiftDriver/Jobs/ReplJob.swift @@ -16,7 +16,7 @@ extension Driver { var inputs: [TypedVirtualPath] = [] try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .repl) - // FIXME: MSVC runtime flags + try addRuntimeLibraryFlags(commandLine: &commandLine) try commandLine.appendLast(.importObjcHeader, from: &parsedOptions) toolchain.addLinkedLibArgs( diff --git a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift index ede3ebc5e..705d28f47 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift @@ -43,7 +43,8 @@ extension DarwinToolchain { addPathEnvironmentVariableIfNeeded("DYLD_FRAMEWORK_PATH", to: &envVars, currentEnv: env, option: .F, - parsedOptions: &parsedOptions) + parsedOptions: &parsedOptions, + extraPaths: ["/System/Library/Frameworks"]) return envVars } diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index 5b73fc1db..f2fad3ad0 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -26,14 +26,15 @@ extension Toolchain { for targetInfo: FrontendTargetInfo, parsedOptions: inout ParsedOptions ) throws -> VirtualPath { - var platform = targetInfo.target.triple.platformName(conflatingDarwin: true)! - // compiler-rt moved these Android sanitizers into `lib/linux/` a couple - // years ago, llvm/llvm-project@a68ccba, so look for them there instead. - if platform == "android" { - platform = "linux" - } + let platform = targetInfo.target.triple.clangOSLibName + + // NOTE(compnerd) Windows uses the per-target runtime directory for the + // Windows runtimes. This should also be done for the other platforms, but + // is not critical. This is done to allow for the Windows runtimes to be + // co-located for all the currently supported architectures: x86, x64, arm64. + let bIsWindows = targetInfo.target.triple.isWindows return VirtualPath.lookup(targetInfo.runtimeResourcePath.path) - .appending(components: "clang", "lib", platform) + .appending(components: "clang", "lib", bIsWindows ? targetInfo.target.triple.triple : platform) } func runtimeLibraryPaths( @@ -90,6 +91,19 @@ extension Toolchain { commandLine.appendFlag(match.option.spelling + match.argument.asSingle) } } + + func addExtraClangLinkerArgs( + to commandLine: inout [Job.ArgTemplate], + parsedOptions: inout ParsedOptions + ) throws { + // Because we invoke `clang` as the linker executable, we must still + // use `-Xlinker` for linker-specific arguments. + for linkerOpt in parsedOptions.arguments(for: .Xlinker) { + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag(linkerOpt.argument.asSingle) + } + try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + } } // MARK: - Common argument routines @@ -117,21 +131,26 @@ extension DarwinToolchain { } } - for compatibilityLib in targetInfo.target.compatibilityLibraries { - let shouldLink: Bool - switch compatibilityLib.filter { - case .all: - shouldLink = true - break + let experimentalFeatures = parsedOptions.arguments(for: .enableExperimentalFeature) + let embeddedEnabled = experimentalFeatures.map(\.argument).map(\.asSingle).contains("Embedded") - case .executable: - shouldLink = linkerOutputType == .executable - } + if !embeddedEnabled { + for compatibilityLib in targetInfo.target.compatibilityLibraries { + let shouldLink: Bool + switch compatibilityLib.filter { + case .all: + shouldLink = true + break - if shouldLink { - // Old frontends don't set forceLoad at all; assume it's true in that case - try addArgsForBackDeployLib("lib" + compatibilityLib.libraryName + ".a", - forceLoad: compatibilityLib.forceLoad ?? true) + case .executable: + shouldLink = linkerOutputType == .executable + } + + if shouldLink { + // Old frontends don't set forceLoad at all; assume it's true in that case + try addArgsForBackDeployLib("lib" + compatibilityLib.libraryName + ".a", + forceLoad: compatibilityLib.forceLoad ?? true) + } } } diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index cacc385ca..1ecb9bd6c 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -11,18 +11,17 @@ //===----------------------------------------------------------------------===// extension Driver { - func computeCacheKeyForInterface(emitModuleJob: Job, - interfaceKind: FileType) throws -> String? { - assert(interfaceKind == .swiftInterface || interfaceKind == .privateSwiftInterface, + mutating func computeCacheKeyForInterface(emitModuleJob: Job, + interfaceKind: FileType) throws -> String? { + assert(interfaceKind == .swiftInterface || interfaceKind == .privateSwiftInterface || interfaceKind == .packageSwiftInterface, "only expect interface output kind") let isNeeded = emitModuleJob.outputs.contains { $0.type == interfaceKind } - guard enableCaching && isNeeded else { return nil } + guard isCachingEnabled && isNeeded else { return nil } // Assume swiftinterface file is always the supplementary output for first input file. - let mainInput = emitModuleJob.inputs[0] - return try interModuleDependencyOracle.computeCacheKeyForOutput(kind: interfaceKind, - commandLine: emitModuleJob.commandLine, - input: mainInput.fileHandle) + let key = try computeOutputCacheKey(commandLine: emitModuleJob.commandLine, + index: 0) + return key } @_spi(Testing) @@ -31,13 +30,13 @@ extension Driver { return isFrontendArgSupported(.inputFileKey) } - mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath, emitModuleJob: Job, optIn: Bool) throws -> Job { + mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath, emitModuleJob: Job, reportAsError: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [interfaceInput] commandLine.appendFlags("-frontend", "-typecheck-module-from-interface") - commandLine.appendPath(interfaceInput.file) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .verifyModuleInterface) - // FIXME: MSVC runtime flags + try addPathArgument(interfaceInput.file, to: &commandLine) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, kind: .verifyModuleInterface, bridgingHeaderHandling: .ignored) + try addRuntimeLibraryFlags(commandLine: &commandLine) // Output serialized diagnostics for this job, if specifically requested var outputs: [TypedVirtualPath] = [] @@ -55,9 +54,14 @@ extension Driver { } // TODO: remove this because we'd like module interface errors to fail the build. - if !optIn && isFrontendArgSupported(.downgradeTypecheckInterfaceError) { + if isFrontendArgSupported(.downgradeTypecheckInterfaceError) && + (!reportAsError || + // package interface is new and should not be a blocker for now + interfaceInput.type == .packageSwiftInterface) { commandLine.appendFlag(.downgradeTypecheckInterfaceError) } + + let cacheKeys = try computeOutputCacheKeyForJob(commandLine: commandLine, inputs: [(interfaceInput, 0)]) return Job( moduleName: moduleOutputInfo.name, kind: .verifyModuleInterface, @@ -66,7 +70,8 @@ extension Driver { displayInputs: [interfaceInput], inputs: inputs, primaryInputs: [], - outputs: outputs + outputs: outputs, + outputCacheKeys: cacheKeys ) } } diff --git a/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift index 4a013a423..aee547b52 100644 --- a/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift @@ -43,7 +43,9 @@ extension WebAssemblyToolchain { commandLine.appendFlag("-fuse-ld=\(linkerArg)") } - try commandLine.appendLast(.ldPath, from: &parsedOptions) + if let arg = parsedOptions.getLastArgument(.ldPath)?.asSingle { + commandLine.append(.joinedOptionAndPath("--ld-path=", try VirtualPath(path: arg))) + } // Configure the toolchain. // @@ -71,6 +73,10 @@ extension WebAssemblyToolchain { if let tool = lookupExecutablePath(filename: "clang", searchPaths: [toolsDir]) { clangPath = tool } + + // Look for binutils in the toolchain folder. + commandLine.appendFlag("-B") + commandLine.appendPath(toolsDir) } guard !parsedOptions.hasArgument(.noStaticStdlib, .noStaticExecutable) else { @@ -84,7 +90,7 @@ extension WebAssemblyToolchain { isShared: false ) - if !parsedOptions.hasArgument(.nostartfiles) { + if !parsedOptions.hasArgument(.nostartfiles) && !parsedOptions.isEmbeddedEnabled { let swiftrtPath = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) .appending( components: targetTriple.platformName() ?? "", @@ -100,6 +106,8 @@ extension WebAssemblyToolchain { return .responseFilePath(input.file) } else if input.type == .object { return .path(input.file) + } else if lto != nil && input.type == .llvmBitcode { + return .path(input.file) } else { return nil } @@ -117,15 +125,24 @@ extension WebAssemblyToolchain { commandLine.appendPath(path) } - // Link the standard library and dependencies. - let linkFilePath: VirtualPath = - VirtualPath.lookup(targetInfo.runtimeResourcePath.path) - .appending(components: targetTriple.platformName() ?? "", - "static-executable-args.lnk") - guard try fileSystem.exists(linkFilePath) else { - throw Error.missingExternalDependency(linkFilePath.name) + let runtimeResourcePath = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) + if parsedOptions.isEmbeddedEnabled { + // Allow linking certain standard library modules (`_Concurrency` etc) + let embeddedLibrariesPath: VirtualPath = runtimeResourcePath.appending( + components: "embedded", targetTriple.triple + ) + commandLine.append(.flag("-Xlinker")) + commandLine.append(.joinedOptionAndPath("-L", embeddedLibrariesPath)) + } else { + // Link the standard library and dependencies. + let linkFilePath: VirtualPath = runtimeResourcePath + .appending(components: targetTriple.platformName() ?? "", + "static-executable-args.lnk") + guard try fileSystem.exists(linkFilePath) else { + throw Error.missingExternalDependency(linkFilePath.name) + } + commandLine.append(.responseFilePath(linkFilePath)) } - commandLine.append(.responseFilePath(linkFilePath)) // Pass down an optimization level if let optArg = mapOptimizationLevelToClangArg(from: &parsedOptions) { @@ -135,14 +152,41 @@ extension WebAssemblyToolchain { // Explicitly pass the target to the linker commandLine.appendFlag("--target=\(targetTriple.triple)") + // WebAssembly doesn't reserve low addresses as its ABI. But without + // "extra inhabitants" of the pointer representation, runtime performance + // and memory footprint are significantly degraded. So we reserve the + // low addresses to use them as extra inhabitants by telling the lowest + // valid address to the linker. + // The value of lowest valid address, called "global base", must be always + // synchronized with `SWIFT_ABI_WASM32_LEAST_VALID_POINTER` defined in + // apple/swift's runtime library. + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag("--global-base=4096") + // Delegate to Clang for sanitizers. It will figure out the correct linker // options. - guard sanitizers.isEmpty else { - throw Error.sanitizersUnsupportedForTarget(targetTriple.triple) + if linkerOutputType == .executable && !sanitizers.isEmpty { + let sanitizerNames = sanitizers + .map { $0.rawValue } + .sorted() // Sort so we get a stable, testable order + .joined(separator: ",") + commandLine.appendFlag("-fsanitize=\(sanitizerNames)") + } + + if parsedOptions.hasArgument(.profileGenerate) { + let libProfile = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) + .appending(components: "clang", "lib", targetTriple.osName, + "libclang_rt.profile-\(targetTriple.archName).a") + commandLine.appendPath(libProfile) } - guard !parsedOptions.hasArgument(.profileGenerate) else { - throw Error.profilingUnsupportedForTarget(targetTriple.triple) + if let lto = lto { + switch lto { + case .llvmFull: + commandLine.appendFlag("-flto=full") + case .llvmThin: + commandLine.appendFlag("-flto=thin") + } } // Run clang++ in verbose mode if "-v" is set @@ -156,8 +200,7 @@ extension WebAssemblyToolchain { from: &parsedOptions ) addLinkedLibArgs(to: &commandLine, parsedOptions: &parsedOptions) - try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) - try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + try addExtraClangLinkerArgs(to: &commandLine, parsedOptions: &parsedOptions) // This should be the last option, for convenience in checking output. commandLine.appendFlag(.o) @@ -168,7 +211,7 @@ extension WebAssemblyToolchain { commandLine.appendFlag("crs") commandLine.appendPath(outputFile) - commandLine.append(contentsOf: inputs.map { .path($0.file) }) + commandLine.append(contentsOf: inputs.lazy.filter { $0.type != .autolink }.map { .path($0.file) }) return try resolvedTool(.staticLinker(lto)) } } diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index 28213b175..eae53e381 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -39,18 +39,47 @@ extension WindowsToolchain { sanitizers: Set, targetInfo: FrontendTargetInfo) throws -> ResolvedTool { + // Check to see whether we need to use lld as the linker. + let bForceLLD: Bool = { + // If LTO is enabled, we need to use lld-link to handle LLVM bitcode. + guard lto == nil else { return true } + + // Profiling currently relies on the ability to emit duplicate weak + // symbols across translation units and having the linker coalesce them. + // Unfortunately link.exe does not support this, so require lld-link + // for now, which supports the behavior via a flag. + // TODO: Once we've changed coverage to no longer rely on emitting + // duplicate weak symbols (rdar://131295678), we can remove this. + if parsedOptions.hasArgument(.profileGenerate) { return true } + + return false + }() + +#if swift(<6.0) + let bUseLLD: Bool + switch parsedOptions.getLastArgument(.useLd)?.asSingle { + case .some("lld"), .some("lld.exe"), .some("lld-link"), .some("lld-link.exe"): + bUseLLD = true + default: + bUseLLD = false + } +#else + let bUseLLD: Bool = switch parsedOptions.getLastArgument(.useLd)?.asSingle { + case .some("lld"), .some("lld.exe"), .some("lld-link"), .some("lld-link.exe"): true + default: false + } +#endif + // Special case static linking as clang cannot drive the operation. if linkerOutputType == .staticLibrary { let librarian: String - switch parsedOptions.getLastArgument(.useLd)?.asSingle { - case .none: - librarian = lto == nil ? "link" : "lld-link" - case .some("lld"), .some("lld.exe"), .some("lld-link"), .some("lld-link.exe"): + if bForceLLD || bUseLLD { librarian = "lld-link" - case let .some(linker): - librarian = linker + } else if let ld = parsedOptions.getLastArgument(.useLd)?.asSingle { + librarian = ld + } else { + librarian = "link" } - commandLine.appendFlag("/LIB") commandLine.appendFlag("/NOLOGO") commandLine.appendFlag("/OUT:\(outputFile.name.spm_shellEscaped())") @@ -62,15 +91,20 @@ extension WindowsToolchain { return try resolvedTool(.staticLinker(lto), pathOverride: lookup(executable: librarian)) } - var cxxCompatEnabled = parsedOptions.hasArgument(.enableExperimentalCxxInterop) - if let cxxInteropMode = parsedOptions.getLastArgument(.cxxInteroperabilityMode) { - if cxxInteropMode.asSingle == "swift-5.9" { - cxxCompatEnabled = true - } - } - let clangTool: Tool = cxxCompatEnabled ? .clangxx : .clang + let enableCxxInterop = + parsedOptions.hasArgument(.enableExperimentalCxxInterop) || + ![nil, "off"].contains(parsedOptions.getLastArgument(.cxxInteroperabilityMode)?.asSingle) + + let clangTool: Tool = enableCxxInterop ? .clangxx : .clang var clang = try getToolPath(clangTool) + // We invoke clang as `clang.exe`, which expects a POSIX-style response file + // by default (`clang-cl.exe` expects Windows-style response files). The + // driver is outputting Windows-style response files because swift-frontend + // expects Windows-style response files. Force `clang.exe` into parsing + // Windows-style response files. + commandLine.appendFlag("--rsp-quoting=windows") + let targetTriple = targetInfo.target.triple if !targetTriple.triple.isEmpty { commandLine.appendFlag("-target") @@ -99,13 +133,15 @@ extension WindowsToolchain { } // Select the linker to use. - if let arg = parsedOptions.getLastArgument(.useLd) { - commandLine.appendFlag("-fuse-ld=\(arg.asSingle)") - } else if lto != nil { + if bForceLLD || bUseLLD { commandLine.appendFlag("-fuse-ld=lld") + } else if let arg = parsedOptions.getLastArgument(.useLd)?.asSingle { + commandLine.appendFlag("-fuse-ld=\(arg)") } - try commandLine.appendLast(.ldPath, from: &parsedOptions) + if let arg = parsedOptions.getLastArgument(.ldPath)?.asSingle { + commandLine.append(.joinedOptionAndPath("--ld-path=", try VirtualPath(path: arg))) + } switch lto { case .some(.llvmThin): @@ -141,7 +177,10 @@ extension WindowsToolchain { // finally falling back to the target information. let rsrc: VirtualPath if let resourceDir = parsedOptions.getLastArgument(.resourceDir) { - rsrc = try VirtualPath(path: resourceDir.asSingle) + rsrc = try VirtualPath(path: AbsolutePath(validating: resourceDir.asSingle) + .appending(components: targetTriple.platformName() ?? "", + architecture(for: targetTriple)) + .pathString) } else if let sdk = parsedOptions.getLastArgument(.sdk)?.asSingle ?? env["SDKROOT"], !sdk.isEmpty { rsrc = try VirtualPath(path: AbsolutePath(validating: sdk) .appending(components: "usr", "lib", "swift", @@ -150,6 +189,8 @@ extension WindowsToolchain { .pathString) } else { rsrc = VirtualPath.lookup(targetInfo.runtimeResourcePath.path) + .appending(components: targetTriple.platformName() ?? "", + architecture(for: targetTriple)) } commandLine.appendPath(rsrc.appending(component: "swiftrt.obj")) } @@ -177,30 +218,28 @@ extension WindowsToolchain { commandLine.appendPath(VirtualPath.lookup(sdkPath)) } - if let stdlib = parsedOptions.getLastArgument(.experimentalCxxStdlib) { - commandLine.appendFlag("-stdlib=\(stdlib.asSingle)") - } - // Pass down an optimization level if let optArg = mapOptimizationLevelToClangArg(from: &parsedOptions) { commandLine.appendFlag(optArg) } - // FIXME(compnerd) render asan/ubsan runtime link for executables + if !sanitizers.isEmpty { + let sanitize = sanitizers.map(\.rawValue).sorted().joined(separator: ",") + commandLine.appendFlag("-fsanitize=\(sanitize)") + } if parsedOptions.contains(.profileGenerate) { - commandLine.appendFlag("-Xlinker") - // FIXME(compnerd) wrap llvm::getInstrProfRuntimeHookVarName() - commandLine.appendFlag("-include:__llvm_profile_runtime") - commandLine.appendFlag("-lclang_rt.profile") + assert(bForceLLD, + "LLD is currently required for profiling (rdar://131295678)") + + commandLine.appendFlag("-fprofile-generate") + // FIXME(rdar://131295678): Currently profiling requires the ability to + // emit duplicate weak symbols. Assume we're using lld and pass + // `-lld-allow-duplicate-weak` to enable this behavior. + commandLine.appendFlags("-Xlinker", "-lld-allow-duplicate-weak") } - for option in parsedOptions.arguments(for: .Xlinker) { - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag(option.argument.asSingle) - } - // TODO(compnerd) is there a separate equivalent to OPT_linker_option_group? - try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + try addExtraClangLinkerArgs(to: &commandLine, parsedOptions: &parsedOptions) if parsedOptions.contains(.v) { commandLine.appendFlag("-v") diff --git a/Sources/SwiftDriver/SwiftDriver.docc/SwiftDriver.md b/Sources/SwiftDriver/SwiftDriver.docc/SwiftDriver.md index b09eaa40e..e24883a55 100644 --- a/Sources/SwiftDriver/SwiftDriver.docc/SwiftDriver.md +++ b/Sources/SwiftDriver/SwiftDriver.docc/SwiftDriver.md @@ -9,7 +9,7 @@ into various compiled results: executables, libraries, object files, Swift modules and interfaces, etc. It is the program one invokes from the command line to build Swift code (i.e., swift or swiftc) and is often invoked on the developer's behalf by a build system such as the -[Swift Package Manager](https://github.com/apple/swift-package-manager) +[Swift Package Manager](https://github.com/swiftlang/swift-package-manager) or Xcode's build system. ## Topics diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift index f639ba108..f55fef5be 100644 --- a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift +++ b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift @@ -53,40 +53,6 @@ internal extension SwiftScan { } return InterModuleDependencyImports(imports: try toSwiftStringArray(importsRef.pointee), moduleAliases: moduleAliases) } - - /// From a reference to a binary-format dependency graph collection returned by libSwiftScan batch scan query, - /// corresponding to the specified batch scan input (`BatchScanModuleInfo`), construct instances of - /// `InterModuleDependencyGraph` for each result. - func constructBatchResultGraphs(for batchInfos: [BatchScanModuleInfo], - moduleAliases: [String: String]?, - from batchResultRef: swiftscan_batch_scan_result_t) throws - -> [ModuleDependencyId: [InterModuleDependencyGraph]] { - var resultMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = [:] - let resultGraphRefArray = Array(UnsafeBufferPointer(start: batchResultRef.results, - count: Int(batchResultRef.count))) - // Note, respective indices of the batch scan input and the returned result must be aligned. - for (index, resultGraphRefOrNull) in resultGraphRefArray.enumerated() { - guard let resultGraphRef = resultGraphRefOrNull else { - throw DependencyScanningError.dependencyScanFailed - } - let decodedGraph = try constructGraph(from: resultGraphRef, moduleAliases: moduleAliases) - - let moduleId: ModuleDependencyId - switch batchInfos[index] { - case .swift(let swiftModuleBatchScanInfo): - moduleId = .swift(swiftModuleBatchScanInfo.swiftModuleName) - case .clang(let clangModuleBatchScanInfo): - moduleId = .clang(clangModuleBatchScanInfo.clangModuleName) - } - // Update the map with either yet another graph or create an entry for this module - if resultMap[moduleId] != nil { - resultMap[moduleId]!.append(decodedGraph) - } else { - resultMap[moduleId] = [decodedGraph] - } - } - return resultMap - } } private extension SwiftScan { @@ -120,6 +86,24 @@ private extension SwiftScan { directDependencies = nil } + var linkLibraries: [LinkLibraryInfo] = [] + if supportsLinkLibraries { + let linkLibrarySetRefOrNull = api.swiftscan_module_info_get_link_libraries(moduleInfoRef) + guard let linkLibrarySetRef = linkLibrarySetRefOrNull else { + throw DependencyScanningError.missingField("dependency_graph.link_libraries") + } + // Turn the `swiftscan_dependency_set_t` into an array of `swiftscan_dependency_info_t` + // references we can iterate through in order to construct `ModuleInfo` objects. + let linkLibraryRefArray = Array(UnsafeBufferPointer(start: linkLibrarySetRef.pointee.link_libraries, + count: Int(linkLibrarySetRef.pointee.count))) + for linkLibraryRefOrNull in linkLibraryRefArray { + guard let linkLibraryRef = linkLibraryRefOrNull else { + throw DependencyScanningError.missingField("dependency_set_t.link_libraries[_]") + } + linkLibraries.append(try constructLinkLibrayInfo(from: linkLibraryRef)) + } + } + guard let moduleDetailsRef = api.swiftscan_module_info_get_details(moduleInfoRef) else { throw DependencyScanningError.missingField("modules[\(moduleId)].details") } @@ -128,9 +112,16 @@ private extension SwiftScan { return (moduleId, ModuleInfo(modulePath: modulePath, sourceFiles: sourceFiles, directDependencies: directDependencies, + linkLibraries: linkLibraries, details: details)) } + func constructLinkLibrayInfo(from linkLibraryInfoRef: swiftscan_link_library_info_t) throws -> LinkLibraryInfo { + return LinkLibraryInfo(linkName: try toSwiftString(api.swiftscan_link_library_info_get_link_name(linkLibraryInfoRef)), + isFramework: api.swiftscan_link_library_info_get_is_framework(linkLibraryInfoRef), + shouldForceLoad: api.swiftscan_link_library_info_get_should_force_load(linkLibraryInfoRef)) + } + /// From a reference to a binary-format module info details object info returned by libSwiftScan, /// construct an instance of an `ModuleInfo`.Details as used by the driver. /// The object returned by libSwiftScan is a union so ensure to execute dependency-specific queries. @@ -187,16 +178,16 @@ private extension SwiftScan { let bridgingPchCommandLine = supportsBridgingHeaderPCHCommand ? try getOptionalStringArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_bridging_pch_command_line) : nil - let extraPcmArgs = - try getStringArrayDetail(from: moduleDetailsRef, - using: api.swiftscan_swift_textual_detail_get_extra_pcm_args, - fieldName: "extraPCMArgs") let contextHash = try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_context_hash) let isFramework = api.swiftscan_swift_textual_detail_get_is_framework(moduleDetailsRef) let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_module_cache_key) : nil + let chainedBridgingHeaderPath = supportsChainedBridgingHeader ? + try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_chained_bridging_header_path) : nil + let chainedBridgingHeaderContent = supportsChainedBridgingHeader ? + try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_textual_detail_get_chained_bridging_header_content) : nil // Decode all dependencies of this module let swiftOverlayDependencies: [ModuleDependencyId]? @@ -209,16 +200,28 @@ private extension SwiftScan { swiftOverlayDependencies = nil } + let sourceImportedDependencies: [ModuleDependencyId]? + if supportsSeparateImportOnlyDependencise, + let encodedImportedDepsRef = api.swiftscan_swift_textual_detail_get_swift_source_import_module_dependencies(moduleDetailsRef) { + let encodedImportedDepsendencies = try toSwiftStringArray(encodedImportedDepsRef.pointee) + sourceImportedDependencies = + try encodedImportedDepsendencies.map { try decodeModuleNameAndKind(from: $0, moduleAliases: moduleAliases) } + } else { + sourceImportedDependencies = nil + } + return SwiftModuleDetails(moduleInterfacePath: moduleInterfacePath, compiledModuleCandidates: compiledModuleCandidates, bridgingHeader: bridgingHeader, commandLine: commandLine, bridgingPchCommandLine : bridgingPchCommandLine, contextHash: contextHash, - extraPcmArgs: extraPcmArgs, isFramework: isFramework, swiftOverlayDependencies: swiftOverlayDependencies, - moduleCacheKey: moduleCacheKey) + sourceImportDependencies: sourceImportedDependencies, + moduleCacheKey: moduleCacheKey, + chainedBridgingHeaderPath: chainedBridgingHeaderPath, + chainedBridgingHeaderContent: chainedBridgingHeaderContent) } /// Construct a `SwiftPrebuiltExternalModuleDetails` from a `swiftscan_module_details_t` reference @@ -239,6 +242,10 @@ private extension SwiftScan { if supportsBinaryModuleHeaderDependencies { headerDependencies = try getOptionalPathArrayDetail(from: moduleDetailsRef, using: api.swiftscan_swift_binary_detail_get_header_dependencies) + } else if supportsBinaryModuleHeaderDependency, + let header = try getOptionalPathDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_header_dependency) { + headerDependencies = [header] } else { headerDependencies = nil } @@ -249,15 +256,22 @@ private extension SwiftScan { } else { isFramework = false } + + let headerDependencyModuleDependencies: [ModuleDependencyId]? = + hasBinarySwiftModuleHeaderModuleDependencies ? + try getOptionalStringArrayDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_header_dependency_module_dependencies)?.map { .clang($0) } : nil + let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_swift_binary_detail_get_module_cache_key) : nil - return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, - moduleDocPath: moduleDocPath, - moduleSourceInfoPath: moduleSourceInfoPath, - headerDependencies: headerDependencies, - isFramework: isFramework, - moduleCacheKey: moduleCacheKey) + return SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, + moduleDocPath: moduleDocPath, + moduleSourceInfoPath: moduleSourceInfoPath, + headerDependencyPaths: headerDependencies, + headerDependencyModuleDependencies: headerDependencyModuleDependencies, + isFramework: isFramework, + moduleCacheKey: moduleCacheKey) } /// Construct a `SwiftPlaceholderModuleDetails` from a `swiftscan_module_details_t` reference @@ -289,22 +303,12 @@ private extension SwiftScan { using: api.swiftscan_clang_detail_get_command_line, fieldName: "clang_detail.commandLine") - let capturedPCMArgs : Set<[String]>? - if clangDetailsHaveCapturedPCMArgs { - let capturedArgs = try getStringArrayDetail(from: moduleDetailsRef, - using: api.swiftscan_clang_detail_get_captured_pcm_args, - fieldName: "clang_detail.capturedPCMArgs") - capturedPCMArgs = [capturedArgs] - } else { - capturedPCMArgs = nil - } let moduleCacheKey = supportsCaching ? try getOptionalStringDetail(from: moduleDetailsRef, using: api.swiftscan_clang_detail_get_module_cache_key) : nil return ClangModuleDetails(moduleMapPath: moduleMapPath, contextHash: contextHash, commandLine: commandLine, - capturedPCMArgs: capturedPCMArgs, moduleCacheKey: moduleCacheKey) } } diff --git a/Sources/SwiftDriver/SwiftScan/Loader.swift b/Sources/SwiftDriver/SwiftScan/Loader.swift index 92ec883fe..decc1d5de 100644 --- a/Sources/SwiftDriver/SwiftScan/Loader.swift +++ b/Sources/SwiftDriver/SwiftScan/Loader.swift @@ -15,12 +15,14 @@ import var Foundation.NSLocalizedDescriptionKey #if os(Windows) import WinSDK -#elseif os(iOS) || os(macOS) || os(tvOS) || os(watchOS) +#elseif canImport(Darwin) import Darwin #elseif canImport(Glibc) import Glibc #elseif canImport(Musl) import Musl +#elseif canImport(Android) +import Android #endif internal enum Loader { @@ -119,7 +121,7 @@ extension Loader.Flags { } // Platform-specific flags -#if os(iOS) || os(macOS) || os(tvOS) || os(watchOS) +#if canImport(Darwin) public static var first: Loader.Flags { Loader.Flags(rawValue: RTLD_FIRST) } @@ -159,6 +161,19 @@ extension Loader { return Handle(value: handle) } + public static func getSelfHandle(mode: Flags) throws -> Handle { +#if os(Windows) + guard let handle = GetModuleHandleW(nil) else { + throw Loader.Error.open("GetModuleHandleW(nil) failure: \(GetLastError())") + } +#else + guard let handle = dlopen(nil, mode.rawValue) else { + throw Loader.Error.open(Loader.error() ?? "unknown error") + } +#endif + return Handle(value: handle) + } + public static func lookup(symbol: String, in module: Handle) -> T? { #if os(Windows) guard let pointer = GetProcAddress(module.value!, symbol) else { diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index a77cb1227..804a1e8d6 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -12,25 +12,31 @@ @_implementationOnly import CSwiftScan +#if os(Windows) +import CRT +#endif + import func Foundation.strdup import func Foundation.free import class Foundation.JSONDecoder +import protocol Foundation.LocalizedError import struct Foundation.Data import protocol TSCBasic.DiagnosticData import struct TSCBasic.AbsolutePath import struct TSCBasic.Diagnostic +import protocol TSCBasic.DiagnosticLocation -public enum DependencyScanningError: Error, DiagnosticData { +public enum DependencyScanningError: LocalizedError, DiagnosticData, Equatable { case missingRequiredSymbol(String) - case dependencyScanFailed + case dependencyScanFailed(String) case failedToInstantiateScanner case casError(String) case missingField(String) case moduleNameDecodeFailure(String) case unsupportedDependencyDetailsKind(Int) case invalidStringPtr - case scanningLibraryInvocationMismatch(AbsolutePath, AbsolutePath) + case scanningLibraryInvocationMismatch(String, String) case scanningLibraryNotFound(AbsolutePath) case argumentQueryFailed @@ -38,14 +44,14 @@ public enum DependencyScanningError: Error, DiagnosticData { switch self { case .missingRequiredSymbol(let symbolName): return "libSwiftScan missing required symbol: '\(symbolName)'" - case .dependencyScanFailed: - return "libSwiftScan dependency scan query failed" + case .dependencyScanFailed(let reason): + return "Dependency scan query failed: `\(reason)`" case .failedToInstantiateScanner: - return "libSwiftScan failed to create scanner instance" + return "Failed to create scanner instance" case .casError(let reason): - return "libSwiftScan CAS error: \(reason)" + return "CAS error: \(reason)" case .missingField(let fieldName): - return "libSwiftScan scan result missing required field: `\(fieldName)`" + return "Scan result missing required field: `\(fieldName)`" case .moduleNameDecodeFailure(let encodedName): return "Failed to decode dependency module name: `\(encodedName)`" case .unsupportedDependencyDetailsKind(let kindRawValue): @@ -53,18 +59,32 @@ public enum DependencyScanningError: Error, DiagnosticData { case .invalidStringPtr: return "Dependency module details contains a corrupted string reference" case .scanningLibraryInvocationMismatch(let path1, let path2): - return "Dependency Scanning library differs across driver invocations: \(path1.description) and \(path2.description)" + return "Dependency Scanning library differs across driver invocations: \(path1) and \(path2)" case .scanningLibraryNotFound(let path): return "Dependency Scanning library not found at path: \(path)" case .argumentQueryFailed: - return "libSwiftScan supported compiler argument query failed" + return "Supported compiler argument query failed" } } + + public var errorDescription: String? { + return self.description + } +} + +public struct ScannerDiagnosticSourceLocation : DiagnosticLocation { + public var description: String { + return "\(bufferIdentifier):\(lineNumber):\(columnNumber)" + } + public let bufferIdentifier: String + public let lineNumber: Int + public let columnNumber: Int } -@_spi(Testing) public struct ScannerDiagnosticPayload { - @_spi(Testing) public let severity: Diagnostic.Behavior - @_spi(Testing) public let message: String +public struct ScannerDiagnosticPayload { + public let severity: Diagnostic.Behavior + public let message: String + public let sourceLocation: ScannerDiagnosticSourceLocation? } internal extension swiftscan_diagnostic_severity_t { @@ -84,10 +104,16 @@ internal extension swiftscan_diagnostic_severity_t { } } +private extension String { + func stripNewline() -> String { + return self.hasSuffix("\n") ? String(self.dropLast()) : self + } +} + /// Wrapper for libSwiftScan, taking care of initialization, shutdown, and dispatching dependency scanning queries. @_spi(Testing) public final class SwiftScan { /// The path to the libSwiftScan dylib. - let path: AbsolutePath + let path: AbsolutePath? /// The handle to the dylib. let dylib: Loader.Handle @@ -98,16 +124,21 @@ internal extension swiftscan_diagnostic_severity_t { /// Instance of a scanner, which maintains shared state across scan queries. let scanner: swiftscan_scanner_t; - /// Optional CAS instance. - var cas: swiftscan_cas_t? = nil - - @_spi(Testing) public init(dylib path: AbsolutePath) throws { + @_spi(Testing) public init(dylib path: AbsolutePath? = nil) throws { self.path = path - #if os(Windows) - self.dylib = try Loader.load(path.pathString, mode: []) - #else - self.dylib = try Loader.load(path.pathString, mode: [.lazy, .local, .first]) - #endif + if let externalPath = path { +#if os(Windows) + self.dylib = try Loader.load(externalPath.pathString, mode: []) +#else + self.dylib = try Loader.load(externalPath.pathString, mode: [.lazy, .local, .first]) +#endif + } else { +#if os(Windows) + self.dylib = try Loader.getSelfHandle(mode: []) +#else + self.dylib = try Loader.getSelfHandle(mode: [.lazy, .local, .first]) +#endif + } self.api = try swiftscan_functions_t(self.dylib) guard let scanner = api.swiftscan_scanner_create() else { throw DependencyScanningError.failedToInstantiateScanner @@ -117,9 +148,6 @@ internal extension swiftscan_diagnostic_severity_t { deinit { api.swiftscan_scanner_dispose(self.scanner) - if let scan_cas = cas { - api.swiftscan_cas_dispose(scan_cas) - } // FIXME: is it safe to dlclose() swiftscan? If so, do that here. // For now, let the handle leak. dylib.leak() @@ -127,7 +155,8 @@ internal extension swiftscan_diagnostic_severity_t { func preScanImports(workingDirectory: AbsolutePath, moduleAliases: [String: String]?, - invocationCommand: [String]) throws -> InterModuleDependencyImports { + invocationCommand: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { // Create and configure the scanner invocation let invocation = api.swiftscan_scan_invocation_create() defer { api.swiftscan_scan_invocation_dispose(invocation) } @@ -143,19 +172,25 @@ internal extension swiftscan_diagnostic_severity_t { let importSetRefOrNull = api.swiftscan_import_set_create(scanner, invocation) guard let importSetRef = importSetRefOrNull else { - throw DependencyScanningError.dependencyScanFailed + throw DependencyScanningError.dependencyScanFailed("Unable to produce import set") + } + defer { api.swiftscan_import_set_dispose(importSetRef) } + + if canQueryPerScanDiagnostics { + let diagnosticsSetRefOrNull = api.swiftscan_import_set_get_diagnostics(importSetRef) + guard let diagnosticsSetRef = diagnosticsSetRefOrNull else { + throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics") + } + diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef) } - let importSet = try constructImportSet(from: importSetRef, with: moduleAliases) - // Free the memory allocated for the in-memory representation of the import set - // returned by the scanner, now that we have translated it. - api.swiftscan_import_set_dispose(importSetRef) - return importSet + return try constructImportSet(from: importSetRef, with: moduleAliases) } func scanDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]?, - invocationCommand: [String]) throws -> InterModuleDependencyGraph { + invocationCommand: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { // Create and configure the scanner invocation let invocation = api.swiftscan_scan_invocation_create() defer { api.swiftscan_scan_invocation_dispose(invocation) } @@ -171,98 +206,37 @@ internal extension swiftscan_diagnostic_severity_t { let graphRefOrNull = api.swiftscan_dependency_graph_create(scanner, invocation) guard let graphRef = graphRefOrNull else { - throw DependencyScanningError.dependencyScanFailed - } - - let dependencyGraph = try constructGraph(from: graphRef, moduleAliases: moduleAliases) - // Free the memory allocated for the in-memory representation of the dependency - // graph returned by the scanner, now that we have translated it into an - // `InterModuleDependencyGraph`. - api.swiftscan_dependency_graph_dispose(graphRef) - return dependencyGraph - } - - func batchScanDependencies(workingDirectory: AbsolutePath, - moduleAliases: [String: String]?, - invocationCommand: [String], - batchInfos: [BatchScanModuleInfo]) - throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { - // Create and configure the scanner invocation - let invocationRef = api.swiftscan_scan_invocation_create() - defer { api.swiftscan_scan_invocation_dispose(invocationRef) } - api.swiftscan_scan_invocation_set_working_directory(invocationRef, - workingDirectory - .description - .cString(using: String.Encoding.utf8)) - withArrayOfCStrings(invocationCommand) { invocationStringArray in - api.swiftscan_scan_invocation_set_argv(invocationRef, - Int32(invocationCommand.count), - invocationStringArray) + throw DependencyScanningError.dependencyScanFailed("Unable to produce dependency graph") } + defer { api.swiftscan_dependency_graph_dispose(graphRef) } - // Create and populate a batch scan input `swiftscan_batch_scan_input_t` - let moduleEntriesPtr = - UnsafeMutablePointer.allocate(capacity: batchInfos.count) - for (index, batchEntryInfo) in batchInfos.enumerated() { - // Create and populate an individual `swiftscan_batch_scan_entry_t` - let entryRef = api.swiftscan_batch_scan_entry_create() - switch batchEntryInfo { - case .clang(let clangEntryInfo): - api.swiftscan_batch_scan_entry_set_module_name(entryRef, - clangEntryInfo.clangModuleName - .cString(using: String.Encoding.utf8)) - api.swiftscan_batch_scan_entry_set_is_swift(entryRef, false) - api.swiftscan_batch_scan_entry_set_arguments(entryRef, clangEntryInfo.arguments - .cString(using: String.Encoding.utf8)) - case .swift(let swiftEntryInfo): - api.swiftscan_batch_scan_entry_set_module_name(entryRef, - swiftEntryInfo.swiftModuleName - .cString(using: String.Encoding.utf8)) - api.swiftscan_batch_scan_entry_set_is_swift(entryRef, true) + if canQueryPerScanDiagnostics { + let diagnosticsSetRefOrNull = api.swiftscan_dependency_graph_get_diagnostics(graphRef) + guard let diagnosticsSetRef = diagnosticsSetRefOrNull else { + throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics") } - (moduleEntriesPtr + index).initialize(to: entryRef) - } - let inputRef = api.swiftscan_batch_scan_input_create() - // Disposing of the input frees memory of the contained entries, as well. - defer { api.swiftscan_batch_scan_input_dispose(inputRef) } - api.swiftscan_batch_scan_input_set_modules(inputRef, Int32(batchInfos.count), - moduleEntriesPtr) - - let batchResultRefOrNull = api.swiftscan_batch_scan_result_create(scanner, - inputRef, - invocationRef) - guard let batchResultRef = batchResultRefOrNull else { - throw DependencyScanningError.dependencyScanFailed + diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef) } - // Translate `swiftscan_batch_scan_result_t` - // into `[ModuleDependencyId: [InterModuleDependencyGraph]]` - let resultGraphMap = try constructBatchResultGraphs(for: batchInfos, - moduleAliases: moduleAliases, - from: batchResultRef.pointee) - // Free the memory allocated for the in-memory representation of the batch scan - // result, now that we have translated it. - api.swiftscan_batch_scan_result_dispose(batchResultRefOrNull) - return resultGraphMap + + return try constructGraph(from: graphRef, moduleAliases: moduleAliases) } @_spi(Testing) public var hasBinarySwiftModuleIsFramework : Bool { api.swiftscan_swift_binary_detail_get_is_framework != nil } - @_spi(Testing) public var canLoadStoreScannerCache : Bool { - api.swiftscan_scanner_cache_load != nil && - api.swiftscan_scanner_cache_serialize != nil && - api.swiftscan_scanner_cache_reset != nil - } - - @_spi(Testing) public var clangDetailsHaveCapturedPCMArgs : Bool { - api.swiftscan_clang_detail_get_captured_pcm_args != nil + @_spi(Testing) public var hasBinarySwiftModuleHeaderModuleDependencies : Bool { + api.swiftscan_swift_binary_detail_get_header_dependency_module_dependencies != nil } @_spi(Testing) public var supportsBinaryModuleHeaderDependencies : Bool { return api.swiftscan_swift_binary_detail_get_header_dependencies != nil } + @_spi(Testing) public var supportsBinaryModuleHeaderDependency : Bool { + return api.swiftscan_swift_binary_detail_get_header_dependency != nil + } + @_spi(Testing) public var supportsStringDispose : Bool { return api.swiftscan_string_dispose != nil } @@ -272,6 +246,10 @@ internal extension swiftscan_diagnostic_severity_t { return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil } + @_spi(Testing) public var supportsSeparateImportOnlyDependencise: Bool { + return api.swiftscan_swift_textual_detail_get_swift_source_import_module_dependencies != nil + } + @_spi(Testing) public var supportsScannerDiagnostics : Bool { return api.swiftscan_scanner_diagnostics_query != nil && api.swiftscan_scanner_diagnostics_reset != nil && @@ -289,10 +267,11 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_cas_options_dispose != nil && api.swiftscan_cas_options_set_ondisk_path != nil && api.swiftscan_cas_options_set_plugin_path != nil && - api.swiftscan_cas_options_set_option != nil && + api.swiftscan_cas_options_set_plugin_option != nil && api.swiftscan_cas_create_from_options != nil && api.swiftscan_cas_dispose != nil && - api.swiftscan_compute_cache_key != nil && + api.swiftscan_cache_compute_key != nil && + api.swiftscan_cache_compute_key_from_input_index != nil && api.swiftscan_cas_store != nil && api.swiftscan_swift_textual_detail_get_module_cache_key != nil && api.swiftscan_swift_binary_detail_get_module_cache_key != nil && @@ -300,47 +279,86 @@ internal extension swiftscan_diagnostic_severity_t { #endif } + @_spi(Testing) public var supportsCASSizeManagement : Bool { +#if os(Windows) + // CAS is currently not supported on Windows hosts. + return false +#else + return api.swiftscan_cas_get_ondisk_size != nil && + api.swiftscan_cas_set_ondisk_size_limit != nil && + api.swiftscan_cas_prune_ondisk_data != nil +#endif + } + @_spi(Testing) public var supportsBridgingHeaderPCHCommand : Bool { return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil } - func serializeScannerCache(to path: AbsolutePath) { - api.swiftscan_scanner_cache_serialize(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var supportsChainedBridgingHeader : Bool { + return api.swiftscan_swift_textual_detail_get_chained_bridging_header_path != nil && + api.swiftscan_swift_textual_detail_get_chained_bridging_header_content != nil } - func loadScannerCache(from path: AbsolutePath) -> Bool { - return api.swiftscan_scanner_cache_load(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var canQueryPerScanDiagnostics : Bool { + return api.swiftscan_dependency_graph_get_diagnostics != nil && + api.swiftscan_import_set_get_diagnostics != nil } - func resetScannerCache() { - api.swiftscan_scanner_cache_reset(scanner) + @_spi(Testing) public var supportsDiagnosticSourceLocations : Bool { + return api.swiftscan_diagnostic_get_source_location != nil && + api.swiftscan_source_location_get_buffer_identifier != nil && + api.swiftscan_source_location_get_line_number != nil && + api.swiftscan_source_location_get_column_number != nil } - @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { + @_spi(Testing) public var supportsLinkLibraries : Bool { + return api.swiftscan_module_info_get_link_libraries != nil && + api.swiftscan_link_library_info_get_link_name != nil && + api.swiftscan_link_library_info_get_is_framework != nil && + api.swiftscan_link_library_info_get_should_force_load != nil + } + + internal func mapToDriverDiagnosticPayload(_ diagnosticSetRef: UnsafeMutablePointer) throws -> [ScannerDiagnosticPayload] { var result: [ScannerDiagnosticPayload] = [] - let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner) - guard let diagnosticSetRef = diagnosticSetRefOrNull else { - // Seems heavy-handed to fail here - // throw DependencyScanningError.dependencyScanFailed - return [] - } - defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) } let diagnosticRefArray = Array(UnsafeBufferPointer(start: diagnosticSetRef.pointee.diagnostics, count: Int(diagnosticSetRef.pointee.count))) - for diagnosticRefOrNull in diagnosticRefArray { guard let diagnosticRef = diagnosticRefOrNull else { - throw DependencyScanningError.dependencyScanFailed + throw DependencyScanningError.dependencyScanFailed("Unable to produce scanner diagnostics") } - let message = try toSwiftString(api.swiftscan_diagnostic_get_message(diagnosticRef)) + let message = try toSwiftString(api.swiftscan_diagnostic_get_message(diagnosticRef)).stripNewline() let severity = api.swiftscan_diagnostic_get_severity(diagnosticRef) - result.append(ScannerDiagnosticPayload(severity: severity.toDiagnosticBehavior(), message: message)) + + var sourceLoc: ScannerDiagnosticSourceLocation? = nil + if supportsDiagnosticSourceLocations { + let sourceLocRefOrNull = api.swiftscan_diagnostic_get_source_location(diagnosticRef) + if let sourceLocRef = sourceLocRefOrNull { + let bufferName = try toSwiftString(api.swiftscan_source_location_get_buffer_identifier(sourceLocRef)) + let lineNumber = api.swiftscan_source_location_get_line_number(sourceLocRef) + let columnNumber = api.swiftscan_source_location_get_column_number(sourceLocRef) + sourceLoc = ScannerDiagnosticSourceLocation(bufferIdentifier: bufferName, + lineNumber: Int(lineNumber), + columnNumber: Int(columnNumber)) + } + } + result.append(ScannerDiagnosticPayload(severity: severity.toDiagnosticBehavior(), + message: message, + sourceLocation: sourceLoc)) } return result } + @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { + let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner) + guard let diagnosticSetRef = diagnosticSetRefOrNull else { + // Seems heavy-handed to fail here + // throw DependencyScanningError.dependencyScanFailed + return [] + } + defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) } + return try mapToDriverDiagnosticPayload(diagnosticSetRef) + } + @_spi(Testing) public func resetScannerDiagnostics() throws { api.swiftscan_scanner_diagnostics_reset(scanner) } @@ -392,16 +410,18 @@ internal extension swiftscan_diagnostic_severity_t { } } - private func handleCASError(_ closure: (inout swiftscan_string_ref_t) -> Bool) throws { + func handleCASError(_ closure: (inout swiftscan_string_ref_t) -> T) throws -> T{ var err_msg : swiftscan_string_ref_t = swiftscan_string_ref_t() - guard !closure(&err_msg) else { + let ret = closure(&err_msg) + if err_msg.length != 0 { let err_str = try toSwiftString(err_msg) api.swiftscan_string_dispose(err_msg) throw DependencyScanningError.casError(err_str) } + return ret } - func createCAS(pluginPath: String?, onDiskPath: String?, pluginOptions: [(String, String)]) throws { + func createCAS(pluginPath: String?, onDiskPath: String?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS { let casOpts = api.swiftscan_cas_options_create() defer { api.swiftscan_cas_options_dispose(casOpts) @@ -414,65 +434,13 @@ internal extension swiftscan_diagnostic_severity_t { } for (name, value) in pluginOptions { try handleCASError { err_msg in - return api.swiftscan_cas_options_set_option(casOpts, name, value, &err_msg) + _ = api.swiftscan_cas_options_set_plugin_option(casOpts, name, value, &err_msg) } } - try handleCASError { err_msg in - self.cas = api.swiftscan_cas_create_from_options(casOpts, &err_msg) - return self.cas == nil - } - } - - func store(data: Data) throws -> String { - guard let scan_cas = self.cas else { - throw DependencyScanningError.casError("cannot store into CAS because CAS is not yet created") - } - let bytes = UnsafeMutablePointer.allocate(capacity: data.count) - data.copyBytes(to: bytes, count: data.count) - var casid: swiftscan_string_ref_t = swiftscan_string_ref_t() - try handleCASError { err_msg in - casid = api.swiftscan_cas_store(scan_cas, bytes, UInt32(data.count), &err_msg) - return casid.data == nil - } - return try toSwiftString(casid) - } - - private func getSwiftScanOutputKind(kind: FileType) -> swiftscan_output_kind_t { - switch (kind) { - case .object: - return SWIFTSCAN_OUTPUT_TYPE_OBJECT - case .swiftModule: - return SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE - case .swiftInterface: - return SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE - case .privateSwiftInterface: - return SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE - case .pcm: - return SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE - case .pch: - return SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH - default: - fatalError("Unsupported") - } - } - - func computeCacheKeyForOutput(kind: FileType, commandLine: [String], input: String) throws -> String { - guard let scan_cas = self.cas else { - throw DependencyScanningError.casError("cannot compute CacheKey for compilation because CAS is not yet created") - } - var casid: swiftscan_string_ref_t = swiftscan_string_ref_t() - try handleCASError { err_msg in - withArrayOfCStrings(commandLine) { commandArray in - casid = api.swiftscan_compute_cache_key(scan_cas, - Int32(commandLine.count), - commandArray, - input.cString(using: String.Encoding.utf8), - getSwiftScanOutputKind(kind: kind), - &err_msg) - } - return casid.data == nil + let cas = try handleCASError { err_msg in + api.swiftscan_cas_create_from_options(casOpts, &err_msg) } - return try toSwiftString(casid) + return SwiftScanCAS(cas: cas!, scanner: self) } } @@ -496,7 +464,7 @@ private extension swiftscan_functions_t { // MARK: Optional Methods // Future optional methods can be queried here - func loadOptional(_ symbol: String) throws -> T? { + func loadOptional(_ symbol: String) -> T? { guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else { return nil } @@ -504,76 +472,126 @@ private extension swiftscan_functions_t { } // Supported features/flags query self.swiftscan_string_set_dispose = - try loadOptional("swiftscan_string_set_dispose") + loadOptional("swiftscan_string_set_dispose") self.swiftscan_compiler_supported_arguments_query = - try loadOptional("swiftscan_compiler_supported_arguments_query") + loadOptional("swiftscan_compiler_supported_arguments_query") self.swiftscan_compiler_supported_features_query = - try loadOptional("swiftscan_compiler_supported_features_query") + loadOptional("swiftscan_compiler_supported_features_query") // Target Info query self.swiftscan_compiler_target_info_query_v2 = - try loadOptional("swiftscan_compiler_target_info_query_v2") - - // Dependency scanner serialization/deserialization features - self.swiftscan_scanner_cache_serialize = - try loadOptional("swiftscan_scanner_cache_serialize") - self.swiftscan_scanner_cache_load = - try loadOptional("swiftscan_scanner_cache_load") - self.swiftscan_scanner_cache_reset = - try loadOptional("swiftscan_scanner_cache_reset") - - // Clang dependency captured PCM args - self.swiftscan_clang_detail_get_captured_pcm_args = - try loadOptional("swiftscan_clang_detail_get_captured_pcm_args") + loadOptional("swiftscan_compiler_target_info_query_v2") // Scanner diagnostic emission query self.swiftscan_scanner_diagnostics_query = - try loadOptional("swiftscan_scanner_diagnostics_query") + loadOptional("swiftscan_scanner_diagnostics_query") self.swiftscan_scanner_diagnostics_reset = - try loadOptional("swiftscan_scanner_diagnostics_reset") + loadOptional("swiftscan_scanner_diagnostics_reset") self.swiftscan_diagnostic_get_message = - try loadOptional("swiftscan_diagnostic_get_message") + loadOptional("swiftscan_diagnostic_get_message") self.swiftscan_diagnostic_get_severity = - try loadOptional("swiftscan_diagnostic_get_severity") + loadOptional("swiftscan_diagnostic_get_severity") self.swiftscan_diagnostics_set_dispose = - try loadOptional("swiftscan_diagnostics_set_dispose") + loadOptional("swiftscan_diagnostics_set_dispose") self.swiftscan_string_dispose = - try loadOptional("swiftscan_string_dispose") + loadOptional("swiftscan_string_dispose") // isFramework on binary module dependencies self.swiftscan_swift_binary_detail_get_is_framework = - try loadOptional("swiftscan_swift_binary_detail_get_is_framework") + loadOptional("swiftscan_swift_binary_detail_get_is_framework") + + // Clang module dependencies of header input of binary module dependencies + self.swiftscan_swift_binary_detail_get_header_dependency_module_dependencies = + loadOptional("swiftscan_swift_binary_detail_get_header_dependency_module_dependencies") + // Bridging PCH build command-line self.swiftscan_swift_textual_detail_get_bridging_pch_command_line = - try loadOptional("swiftscan_swift_textual_detail_get_bridging_pch_command_line") + loadOptional("swiftscan_swift_textual_detail_get_bridging_pch_command_line") + self.swiftscan_swift_textual_detail_get_chained_bridging_header_path = + loadOptional("swiftscan_swift_textual_detail_get_chained_bridging_header_path") + self.swiftscan_swift_textual_detail_get_chained_bridging_header_content = + loadOptional("swiftscan_swift_textual_detail_get_chained_bridging_header_content") // Caching related APIs. self.swiftscan_swift_textual_detail_get_module_cache_key = - try loadOptional("swiftscan_swift_textual_detail_get_module_cache_key") + loadOptional("swiftscan_swift_textual_detail_get_module_cache_key") self.swiftscan_swift_binary_detail_get_module_cache_key = - try loadOptional("swiftscan_swift_binary_detail_get_module_cache_key") + loadOptional("swiftscan_swift_binary_detail_get_module_cache_key") self.swiftscan_clang_detail_get_module_cache_key = - try loadOptional("swiftscan_clang_detail_get_module_cache_key") - - self.swiftscan_cas_options_create = try loadOptional("swiftscan_cas_options_create") - self.swiftscan_cas_options_set_plugin_path = try loadOptional("swiftscan_cas_options_set_plugin_path") - self.swiftscan_cas_options_set_ondisk_path = try loadOptional("swiftscan_cas_options_set_ondisk_path") - self.swiftscan_cas_options_set_option = try loadOptional("swiftscan_cas_options_set_option") - self.swiftscan_cas_options_dispose = try loadOptional("swiftscan_cas_options_dispose") - self.swiftscan_cas_create_from_options = try loadOptional("swiftscan_cas_create_from_options") - self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") - self.swiftscan_compute_cache_key = - try loadOptional("swiftscan_compute_cache_key") - self.swiftscan_cas_store = try loadOptional("swiftscan_cas_store") - + loadOptional("swiftscan_clang_detail_get_module_cache_key") + + self.swiftscan_cas_options_create = loadOptional("swiftscan_cas_options_create") + self.swiftscan_cas_options_set_plugin_path = loadOptional("swiftscan_cas_options_set_plugin_path") + self.swiftscan_cas_options_set_ondisk_path = loadOptional("swiftscan_cas_options_set_ondisk_path") + self.swiftscan_cas_options_set_plugin_option = loadOptional("swiftscan_cas_options_set_plugin_option") + self.swiftscan_cas_options_dispose = loadOptional("swiftscan_cas_options_dispose") + self.swiftscan_cas_create_from_options = loadOptional("swiftscan_cas_create_from_options") + self.swiftscan_cas_get_ondisk_size = loadOptional("swiftscan_cas_get_ondisk_size") + self.swiftscan_cas_set_ondisk_size_limit = loadOptional("swiftscan_cas_set_ondisk_size_limit") + self.swiftscan_cas_prune_ondisk_data = loadOptional("swiftscan_cas_prune_ondisk_data") + self.swiftscan_cas_dispose = loadOptional("swiftscan_cas_dispose") + self.swiftscan_cache_compute_key = loadOptional("swiftscan_cache_compute_key") + self.swiftscan_cache_compute_key_from_input_index = loadOptional("swiftscan_cache_compute_key_from_input_index") + self.swiftscan_cas_store = loadOptional("swiftscan_cas_store") + + self.swiftscan_cache_query = loadOptional("swiftscan_cache_query") + self.swiftscan_cache_query_async = loadOptional("swiftscan_cache_query_async") + + self.swiftscan_cached_compilation_get_num_outputs = loadOptional("swiftscan_cached_compilation_get_num_outputs") + self.swiftscan_cached_compilation_get_output = loadOptional("swiftscan_cached_compilation_get_output") + self.swiftscan_cached_compilation_make_global_async = loadOptional("swiftscan_cached_compilation_make_global_async") + self.swiftscan_cached_compilation_is_uncacheable = loadOptional("swiftscan_cached_compilation_is_uncacheable") + self.swiftscan_cached_compilation_dispose = loadOptional("swiftscan_cached_compilation_dispose") + + self.swiftscan_cached_output_load = loadOptional("swiftscan_cached_output_load") + self.swiftscan_cached_output_load_async = loadOptional("swiftscan_cached_output_load_async") + self.swiftscan_cached_output_is_materialized = loadOptional("swiftscan_cached_output_is_materialized") + self.swiftscan_cached_output_get_casid = loadOptional("swiftscan_cached_output_get_casid") + self.swiftscan_cached_output_get_name = loadOptional("swiftscan_cached_output_get_name") + self.swiftscan_cached_output_dispose = loadOptional("swiftscan_cached_output_dispose") + + self.swiftscan_cache_action_cancel = loadOptional("swiftscan_cache_action_cancel") + self.swiftscan_cache_cancellation_token_dispose = loadOptional("swiftscan_cache_cancellation_token_dispose") + + self.swiftscan_cache_download_cas_object_async = loadOptional("swiftscan_cache_download_cas_object_async") + + self.swiftscan_cache_replay_instance_create = loadOptional("swiftscan_cache_replay_instance_create") + self.swiftscan_cache_replay_instance_dispose = loadOptional("swiftscan_cache_replay_instance_dispose") + self.swiftscan_cache_replay_compilation = loadOptional("swiftscan_cache_replay_compilation") + + self.swiftscan_cache_replay_result_get_stdout = loadOptional("swiftscan_cache_replay_result_get_stdout") + self.swiftscan_cache_replay_result_get_stderr = loadOptional("swiftscan_cache_replay_result_get_stderr") + self.swiftscan_cache_replay_result_dispose = loadOptional("swiftscan_cache_replay_result_dispose") + + self.swiftscan_diagnostic_get_source_location = loadOptional("swiftscan_diagnostic_get_source_location") + self.swiftscan_source_location_get_buffer_identifier = loadOptional("swiftscan_source_location_get_buffer_identifier") + self.swiftscan_source_location_get_line_number = loadOptional("swiftscan_source_location_get_line_number") + self.swiftscan_source_location_get_column_number = loadOptional("swiftscan_source_location_get_column_number") + + self.swiftscan_module_info_get_link_libraries = loadOptional("swiftscan_module_info_get_link_libraries") + self.swiftscan_link_library_info_get_link_name = loadOptional("swiftscan_link_library_info_get_link_name") + self.swiftscan_link_library_info_get_is_framework = loadOptional("swiftscan_link_library_info_get_is_framework") + self.swiftscan_link_library_info_get_should_force_load = loadOptional("swiftscan_link_library_info_get_should_force_load") // Swift Overlay Dependencies self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies = - try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") + loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") + + // Directly-imported source dependencies + self.swiftscan_swift_textual_detail_get_swift_source_import_module_dependencies = + loadOptional("swiftscan_swift_textual_detail_get_swift_source_import_module_dependencies") // Header dependencies of binary modules self.swiftscan_swift_binary_detail_get_header_dependencies = - try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies") + loadOptional("swiftscan_swift_binary_detail_get_header_dependencies") + self.swiftscan_swift_binary_detail_get_header_dependency = + loadOptional("swiftscan_swift_binary_detail_get_header_dependency") + + // Per-scan-query diagnostic output + self.swiftscan_dependency_graph_get_diagnostics = + loadOptional("swiftscan_dependency_graph_get_diagnostics") + self.swiftscan_import_set_get_diagnostics = + loadOptional("swiftscan_import_set_get_diagnostics") // MARK: Required Methods func loadRequired(_ symbol: String) throws -> T { @@ -597,24 +615,6 @@ private extension swiftscan_functions_t { try loadRequired("swiftscan_scan_invocation_create") self.swiftscan_import_set_get_imports = try loadRequired("swiftscan_import_set_get_imports") - self.swiftscan_batch_scan_entry_create = - try loadRequired("swiftscan_batch_scan_entry_create") - self.swiftscan_batch_scan_entry_get_is_swift = - try loadRequired("swiftscan_batch_scan_entry_get_is_swift") - self.swiftscan_batch_scan_entry_get_arguments = - try loadRequired("swiftscan_batch_scan_entry_get_arguments") - self.swiftscan_batch_scan_entry_get_module_name = - try loadRequired("swiftscan_batch_scan_entry_get_module_name") - self.swiftscan_batch_scan_entry_set_is_swift = - try loadRequired("swiftscan_batch_scan_entry_set_is_swift") - self.swiftscan_batch_scan_entry_set_arguments = - try loadRequired("swiftscan_batch_scan_entry_set_arguments") - self.swiftscan_batch_scan_entry_set_module_name = - try loadRequired("swiftscan_batch_scan_entry_set_module_name") - self.swiftscan_batch_scan_input_set_modules = - try loadRequired("swiftscan_batch_scan_input_set_modules") - self.swiftscan_batch_scan_input_create = - try loadRequired("swiftscan_batch_scan_input_create") self.swiftscan_clang_detail_get_command_line = try loadRequired("swiftscan_clang_detail_get_command_line") self.swiftscan_clang_detail_get_context_hash = @@ -665,8 +665,6 @@ private extension swiftscan_functions_t { try loadRequired("swiftscan_swift_textual_detail_get_bridging_module_dependencies") self.swiftscan_swift_textual_detail_get_command_line = try loadRequired("swiftscan_swift_textual_detail_get_command_line") - self.swiftscan_swift_textual_detail_get_extra_pcm_args = - try loadRequired("swiftscan_swift_textual_detail_get_extra_pcm_args") self.swiftscan_scan_invocation_get_argc = try loadRequired("swiftscan_scan_invocation_get_argc") self.swiftscan_scan_invocation_get_argv = @@ -675,32 +673,54 @@ private extension swiftscan_functions_t { try loadRequired("swiftscan_dependency_graph_dispose") self.swiftscan_import_set_dispose = try loadRequired("swiftscan_import_set_dispose") - self.swiftscan_batch_scan_entry_dispose = - try loadRequired("swiftscan_batch_scan_entry_dispose") - self.swiftscan_batch_scan_input_dispose = - try loadRequired("swiftscan_batch_scan_input_dispose") - self.swiftscan_batch_scan_result_dispose = - try loadRequired("swiftscan_batch_scan_result_dispose") self.swiftscan_scan_invocation_dispose = try loadRequired("swiftscan_scan_invocation_dispose") self.swiftscan_dependency_graph_create = try loadRequired("swiftscan_dependency_graph_create") - self.swiftscan_batch_scan_result_create = - try loadRequired("swiftscan_batch_scan_result_create") self.swiftscan_import_set_create = try loadRequired("swiftscan_import_set_create") } } -// TODO: Move to TSC? +// TODO: Move the following functions to TSC? +/// Helper function to scan a sequence type to help generate pointers for C String Arrays. +func scan< + S: Sequence, U +>(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] { + var result: [U] = [] + result.reserveCapacity(seq.underestimatedCount) + var runningResult = initial + for element in seq { + runningResult = combine(runningResult, element) + result.append(runningResult) + } + return result +} + /// Perform an `action` passing it a `const char **` constructed out of `[String]` -func withArrayOfCStrings(_ strings: [String], - _ action: (UnsafeMutablePointer?>?) -> Void) -{ - let cstrings = strings.map { strdup($0) } + [nil] - let unsafeCStrings = cstrings.map { UnsafePointer($0) } - let _ = unsafeCStrings.withUnsafeBufferPointer { - action(UnsafeMutablePointer(mutating: $0.baseAddress)) - } - for ptr in cstrings { if let ptr = ptr { free(ptr) } } +@_spi(Testing) public func withArrayOfCStrings( + _ args: [String], + _ body: (UnsafeMutablePointer?>?) -> T +) -> T { + let argsCounts = Array(args.map { $0.utf8.count + 1 }) + let argsOffsets = [0] + scan(argsCounts, 0, +) + let argsBufferSize = argsOffsets.last! + var argsBuffer: [UInt8] = [] + argsBuffer.reserveCapacity(argsBufferSize) + for arg in args { + argsBuffer.append(contentsOf: arg.utf8) + argsBuffer.append(0) + } + return argsBuffer.withUnsafeMutableBufferPointer { + (argsBuffer) in + let ptr = UnsafeRawPointer(argsBuffer.baseAddress!).bindMemory( + to: Int8.self, capacity: argsBuffer.count) + var cStrings: [UnsafePointer?] = argsOffsets.map { ptr + $0 } + cStrings[cStrings.count - 1] = nil + return cStrings.withUnsafeMutableBufferPointer { + let unsafeString = UnsafeMutableRawPointer($0.baseAddress!).bindMemory( + to: UnsafePointer?.self, capacity: $0.count) + return body(unsafeString) + } + } } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift new file mode 100644 index 000000000..188c26ed9 --- /dev/null +++ b/Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift @@ -0,0 +1,436 @@ +//===------------------------ SwiftScanCAS.swift --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import CSwiftScan +import struct Foundation.Data + +// Swift Package Manager is building with `-disable-implicit-concurrency-module-import` +// to avoid warnings on old SDKs. Explicity importing concurrency if available +// and only adds async APIs when concurrency is available. +#if canImport(_Concurrency) +import _Concurrency +#endif + +public final class CachedCompilation { + let ptr: swiftscan_cached_compilation_t + private let lib: SwiftScan + + init(_ ptr: swiftscan_cached_compilation_t, lib: SwiftScan) { + self.ptr = ptr + self.lib = lib + } + + public lazy var count: UInt32 = { + lib.api.swiftscan_cached_compilation_get_num_outputs(ptr) + }() + + public var isUncacheable: Bool { + lib.api.swiftscan_cached_compilation_is_uncacheable(ptr) + } + + public func makeGlobal(_ callback: @escaping (Swift.Error?) -> ()) { + class CallbackContext { + func retain() -> UnsafeMutableRawPointer { + return Unmanaged.passRetained(self).toOpaque() + } + + let comp: CachedCompilation + let callback: (Swift.Error?) -> () + init(_ compilation: CachedCompilation, _ callback: @escaping (Swift.Error?) -> ()) { + self.comp = compilation + self.callback = callback + } + } + + func callbackFunc(_ context: UnsafeMutableRawPointer?, _ error: swiftscan_string_ref_t) { + let obj = Unmanaged.fromOpaque(context!).takeRetainedValue() + if error.length != 0 { + if let err = try? obj.comp.lib.toSwiftString(error) { + obj.callback(DependencyScanningError.casError(err)) + } else { + obj.callback(DependencyScanningError.casError("unknown makeGlobal error")) + } + } else { + obj.callback(nil) + } + } + + let context = CallbackContext(self, callback) + lib.api.swiftscan_cached_compilation_make_global_async(ptr, context.retain(), callbackFunc, nil) + } + + deinit { + lib.api.swiftscan_cached_compilation_dispose(ptr) + } +} + +extension CachedCompilation: Sequence { + public typealias Element = CachedOutput + public struct Iterator: IteratorProtocol { + public typealias Element = CachedOutput + let limit: UInt32 + let ptr: swiftscan_cached_compilation_t + let lib: SwiftScan + var idx: UInt32 = 0 + public mutating func next() -> CachedOutput? { + guard idx < self.limit else { return nil } + let output = self.lib.api.swiftscan_cached_compilation_get_output(self.ptr, idx) + idx += 1 + // output can never be nil. + return CachedOutput(output!, lib: self.lib) + } + } + public func makeIterator() -> Iterator { + return Iterator(limit: self.count, ptr: self.ptr, lib: self.lib) + } +} + +public final class CachedOutput { + let ptr: swiftscan_cached_output_t + private let lib: SwiftScan + + init(_ ptr: swiftscan_cached_output_t, lib: SwiftScan) { + self.ptr = ptr + self.lib = lib + } + + public func load() throws -> Bool { + try lib.handleCASError { err_msg in + lib.api.swiftscan_cached_output_load(ptr, &err_msg) + } + } + + public var isMaterialized: Bool { + lib.api.swiftscan_cached_output_is_materialized(ptr) + } + + public func getCASID() throws -> String { + let id = lib.api.swiftscan_cached_output_get_casid(ptr) + defer { lib.api.swiftscan_string_dispose(id) } + return try lib.toSwiftString(id) + } + + public func getOutputKindName() throws -> String { + let kind = lib.api.swiftscan_cached_output_get_name(ptr) + defer { lib.api.swiftscan_string_dispose(kind) } + return try lib.toSwiftString(kind) + } + + deinit { + lib.api.swiftscan_cached_output_dispose(ptr) + } +} + +public final class CacheReplayInstance { + let ptr: swiftscan_cache_replay_instance_t + private let lib: SwiftScan + + init(_ ptr: swiftscan_cache_replay_instance_t, lib: SwiftScan) { + self.ptr = ptr + self.lib = lib + } + + deinit { + lib.api.swiftscan_cache_replay_instance_dispose(ptr) + } +} + +public final class CacheReplayResult { + let ptr: swiftscan_cache_replay_result_t + private let lib: SwiftScan + + init(_ ptr: swiftscan_cache_replay_result_t, lib: SwiftScan) { + self.ptr = ptr + self.lib = lib + } + + public func getStdOut() throws -> String { + let str = lib.api.swiftscan_cache_replay_result_get_stdout(ptr) + return try lib.toSwiftString(str) + } + + public func getStdErr() throws -> String { + let str = lib.api.swiftscan_cache_replay_result_get_stderr(ptr) + return try lib.toSwiftString(str) + } + + deinit { + lib.api.swiftscan_cache_replay_result_dispose(ptr) + } +} + +public final class SwiftScanCAS { + let cas: swiftscan_cas_t + private var scanner: SwiftScan! + deinit { + // FIXME: `cas` needs to be disposed after `scanner`. This is because `scanner` contains a separate + // CAS instance contained in `clang::CASOptions` but `cas` is the one exposed to the build system + // and the one that a size limit is set on. When the `scanner` is disposed last then it's the last + // instance closing the database and it doesn't impose any size limit. + // + // This is extremely fragile, a proper fix would be to either eliminate the extra CAS instance + // from `scanner` or have the `scanner`'s CAS instance exposed to the build system. + let swiftscan_cas_dispose = scanner.api.swiftscan_cas_dispose! + scanner = nil + swiftscan_cas_dispose(cas) + } + + init(cas: swiftscan_cas_t, scanner: SwiftScan) { + self.cas = cas + self.scanner = scanner + } + + private func convert(compilation: swiftscan_cached_compilation_t?) -> CachedCompilation? { + return compilation?.convert(scanner) + } + private func convert(instance: swiftscan_cache_replay_instance_t?) -> CacheReplayInstance? { + return instance?.convert(scanner) + } + private func convert(result: swiftscan_cache_replay_result_t?) -> CacheReplayResult? { + return result?.convert(scanner) + } + + public func store(data: Data) throws -> String { + let bytes = UnsafeMutablePointer.allocate(capacity: data.count) + data.copyBytes(to: bytes, count: data.count) + let casid = try scanner.handleCASError { err_msg in + scanner.api.swiftscan_cas_store(cas, bytes, UInt32(data.count), &err_msg) + } + return try scanner.toSwiftString(casid) + } + + public var supportsSizeManagement: Bool { + scanner.supportsCASSizeManagement + } + + public func getStorageSize() throws -> Int64? { + let size = try scanner.handleCASError { err_msg in + scanner.api.swiftscan_cas_get_ondisk_size(cas, &err_msg) + } + return size == -1 ? nil : size + } + + public func setSizeLimit(_ size: Int64) throws { + _ = try scanner.handleCASError { err_msg in + scanner.api.swiftscan_cas_set_ondisk_size_limit(cas, size, &err_msg) + } + } + + public func prune() throws { + _ = try scanner.handleCASError { err_msg in + scanner.api.swiftscan_cas_prune_ondisk_data(cas, &err_msg) + } + } + + @available(*, deprecated) + public func computeCacheKey(commandLine: [String], input: String) throws -> String { + let casid = try scanner.handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + scanner.api.swiftscan_cache_compute_key(cas, + Int32(commandLine.count), + commandArray, + input.cString(using: String.Encoding.utf8), + &err_msg) + } + } + return try scanner.toSwiftString(casid) + } + + public func computeCacheKey(commandLine: [String], index: Int) throws -> String { + let casid = try scanner.handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + scanner.api.swiftscan_cache_compute_key_from_input_index(cas, + Int32(commandLine.count), + commandArray, + UInt32(index), + &err_msg) + } + } + return try scanner.toSwiftString(casid) + } + + public func createReplayInstance(commandLine: [String]) throws -> CacheReplayInstance { + let instance = try scanner.handleCASError { err_msg in + withArrayOfCStrings(commandLine) { commandArray in + scanner.api.swiftscan_cache_replay_instance_create(Int32(commandLine.count), + commandArray, + &err_msg) + } + } + // Never return nullptr when no error occurs. + guard let result = convert(instance: instance) else { + throw DependencyScanningError.casError("unexpected nil for replay instance") + } + return result + } + + public func queryCacheKey(_ key: String, globally: Bool) throws -> CachedCompilation? { + let result = try scanner.handleCASError { error in + scanner.api.swiftscan_cache_query(cas, key.cString(using: .utf8), globally, &error) + } + return convert(compilation: result) + } + + public func replayCompilation(instance: CacheReplayInstance, compilation: CachedCompilation) throws -> CacheReplayResult { + let result = try scanner.handleCASError { err_msg in + scanner.api.swiftscan_cache_replay_compilation(instance.ptr, compilation.ptr, &err_msg) + } + guard let res = convert(result: result) else { + throw DependencyScanningError.casError("unexpected nil for cache_replay_result") + } + return res + } +} + +extension SwiftScanCAS: Equatable { + static public func == (lhs: SwiftScanCAS, rhs: SwiftScanCAS) -> Bool { + return lhs.cas == rhs.cas + } +} + +extension swiftscan_cached_compilation_t { + func convert(_ lib: SwiftScan) -> CachedCompilation { + return CachedCompilation(self, lib: lib) + } +} + +extension swiftscan_cache_replay_instance_t { + func convert(_ lib: SwiftScan) -> CacheReplayInstance { + return CacheReplayInstance(self, lib: lib) + } +} + +extension swiftscan_cache_replay_result_t { + func convert(_ lib: SwiftScan) -> CacheReplayResult { + return CacheReplayResult(self, lib: lib) + } +} + +#if canImport(_Concurrency) +// Async API Vendor +extension CachedCompilation { + public func makeGlobal() async throws { + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + makeGlobal { (error: Swift.Error?) in + if let err = error { + continuation.resume(throwing: err) + } else { + continuation.resume(returning: ()) + } + } + } + } +} + +extension CachedOutput { + public func load() async throws -> Bool { + class CallbackContext { + func retain() -> UnsafeMutableRawPointer { + return Unmanaged.passRetained(self).toOpaque() + } + + let continuation: CheckedContinuation + let output: CachedOutput + init(_ continuation: CheckedContinuation, output: CachedOutput) { + self.continuation = continuation + self.output = output + } + } + + func callbackFunc(_ context: UnsafeMutableRawPointer?, _ success: Bool, _ error: swiftscan_string_ref_t) { + let obj = Unmanaged.fromOpaque(context!).takeRetainedValue() + if error.length != 0 { + if let err = try? obj.output.lib.toSwiftString(error) { + obj.continuation.resume(throwing: DependencyScanningError.casError(err)) + } else { + obj.continuation.resume(throwing: DependencyScanningError.casError("unknown output loading error")) + } + } else { + obj.continuation.resume(returning: success) + } + } + + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let context = CallbackContext(continuation, output: self) + lib.api.swiftscan_cached_output_load_async(ptr, context.retain(), callbackFunc, nil) + } + } +} + +extension SwiftScanCAS { + public func queryCacheKey(_ key: String, globally: Bool) async throws -> CachedCompilation? { + class CallbackContext { + func retain() -> UnsafeMutableRawPointer { + return Unmanaged.passRetained(self).toOpaque() + } + + let continuation: CheckedContinuation + let cas: SwiftScanCAS + init(_ continuation: CheckedContinuation, cas: SwiftScanCAS) { + self.continuation = continuation + self.cas = cas + } + } + + func callbackFunc(_ context: UnsafeMutableRawPointer?, _ comp: swiftscan_cached_compilation_t?, _ error: swiftscan_string_ref_t) { + let obj = Unmanaged.fromOpaque(context!).takeRetainedValue() + if error.length != 0 { + if let err = try? obj.cas.scanner.toSwiftString(error) { + obj.continuation.resume(throwing: DependencyScanningError.casError(err)) + } else { + obj.continuation.resume(throwing: DependencyScanningError.casError("unknown cache querying error")) + } + } else { + obj.continuation.resume(returning: obj.cas.convert(compilation: comp)) + } + } + + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let context = CallbackContext(continuation, cas: self) + scanner.api.swiftscan_cache_query_async(cas, key.cString(using: .utf8), globally, context.retain(), callbackFunc, nil) + } + } + + public func download(with id: String) async throws -> Bool { + class CallbackContext { + func retain() -> UnsafeMutableRawPointer { + return Unmanaged.passRetained(self).toOpaque() + } + + let continuation: CheckedContinuation + let cas: SwiftScanCAS + init(_ continuation: CheckedContinuation, cas: SwiftScanCAS) { + self.continuation = continuation + self.cas = cas + } + } + + func callbackFunc(_ context: UnsafeMutableRawPointer?, _ success: Bool, _ error: swiftscan_string_ref_t) { + let obj = Unmanaged.fromOpaque(context!).takeRetainedValue() + if error.length != 0 { + if let err = try? obj.cas.scanner.toSwiftString(error) { + obj.continuation.resume(throwing: DependencyScanningError.casError(err)) + } else { + obj.continuation.resume(throwing: DependencyScanningError.casError("unknown output loading error")) + } + } else { + obj.continuation.resume(returning: success) + } + } + + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let context = CallbackContext(continuation, cas: self) + scanner.api.swiftscan_cache_download_cas_object_async(cas, id.cString(using: .utf8), context.retain(), callbackFunc, nil) + } + } +} +#endif diff --git a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift index fd523655c..cfb4c9f05 100644 --- a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift @@ -114,6 +114,20 @@ public final class DarwinToolchain: Toolchain { } } + public func addAutoLinkFlags(for linkLibraries: [LinkLibraryInfo], to commandLine: inout [Job.ArgTemplate]) { + for linkLibrary in linkLibraries { + if !linkLibrary.isFramework { + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag("-possible-l\(linkLibrary.linkName)") + } else { + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag("-possible_framework") + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag(linkLibrary.linkName) + } + } + } + public func defaultSDKPath(_ target: Triple?) throws -> AbsolutePath? { let hostIsMacOS: Bool #if os(macOS) @@ -165,7 +179,6 @@ public final class DarwinToolchain: Toolchain { case argumentNotSupported(String) case invalidDeploymentTargetForIR(platform: DarwinPlatform, version: Triple.Version, archName: String) case unsupportedTargetVariant(variant: Triple) - case darwinOnlySupportsLibCxx public var description: String { switch self { @@ -177,8 +190,6 @@ public final class DarwinToolchain: Toolchain { return "unsupported '\(variant.isiOS ? "-target-variant" : "-target")' value '\(variant.triple)'; use 'ios-macabi' instead" case .argumentNotSupported(let argument): return "\(argument) is no longer supported for Apple platforms" - case .darwinOnlySupportsLibCxx: - return "The only C++ standard library supported on Apple platforms is libc++" } } } @@ -211,12 +222,28 @@ public final class DarwinToolchain: Toolchain { if parsedOptions.hasArgument(.staticExecutable) { throw ToolchainValidationError.argumentNotSupported("-static-executable") } - // If a C++ standard library is specified, it has to be libc++. - if let cxxLib = parsedOptions.getLastArgument(.experimentalCxxStdlib) { - if cxxLib.asSingle != "libc++" { - throw ToolchainValidationError.darwinOnlySupportsLibCxx - } + } + + public func getDefaultDwarfVersion(targetTriple: Triple) -> UInt8 { + // Default to DWARF 2 on OS X 10.10 / iOS 8 and lower. + // Default to DWARF 4 on OS X 10.11 - macOS 14 / iOS - iOS 17. + if (targetTriple.isMacOSX && targetTriple.version(for: .macOS) < Triple.Version(10, 11, 0)) || + (targetTriple.isiOS && targetTriple.version( + for: .iOS(targetTriple._isSimulatorEnvironment ? .simulator : .device)) < Triple.Version(9, 0, 0)) { + return 2; + } + if (targetTriple.isMacOSX && targetTriple.version(for: .macOS) < Triple.Version(15, 0, 0)) || + (targetTriple.isiOS && targetTriple.version( + for: .iOS(targetTriple._isSimulatorEnvironment ? .simulator : .device)) < Triple.Version(18, 0, 0)) || + (targetTriple.isTvOS && targetTriple.version( + for: .tvOS(targetTriple._isSimulatorEnvironment ? .simulator : .device)) < Triple.Version(18, 0, 0)) || + (targetTriple.isWatchOS && targetTriple.version( + for: .watchOS(targetTriple._isSimulatorEnvironment ? .simulator : .device)) < Triple.Version(11, 0, 0)) || + (targetTriple.isVisionOS && targetTriple.version( + for: .visionOS(targetTriple._isSimulatorEnvironment ? .simulator : .device)) < Triple.Version(2, 0, 0)){ + return 4 } + return 5 } func validateDeploymentTarget(_ parsedOptions: inout ParsedOptions, @@ -225,12 +252,18 @@ public final class DarwinToolchain: Toolchain { return } + // Embedded Swift should accept all target triples / OS versions / arch combinations + guard !parsedOptions.isEmbeddedEnabled else { + return + } + // Check minimum supported OS versions. Note that Mac Catalyst falls into the iOS device case. The driver automatically uplevels the deployment target to iOS >= 13.1. let minVersions: [Triple.OS: (DarwinPlatform, Triple.Version)] = [ .macosx: (.macOS, Triple.Version(10, 9, 0)), .ios: (.iOS(targetTriple._isSimulatorEnvironment ? .simulator : .device), Triple.Version(7, 0, 0)), .tvos: (.tvOS(targetTriple._isSimulatorEnvironment ? .simulator : .device), Triple.Version(9, 0, 0)), - .watchos: (.watchOS(targetTriple._isSimulatorEnvironment ? .simulator : .device), Triple.Version(2, 0, 0)) + .watchos: (.watchOS(targetTriple._isSimulatorEnvironment ? .simulator : .device), Triple.Version(2, 0, 0)), + .visionos: (.visionOS(targetTriple._isSimulatorEnvironment ? .simulator : .device), Triple.Version(1, 0, 0)) ] if let (platform, minVersion) = minVersions[os], targetTriple.version(for: platform) < minVersion { throw ToolchainValidationError.osVersionBelowMinimumDeploymentTarget(platform: platform, version: minVersion) @@ -279,6 +312,8 @@ public final class DarwinToolchain: Toolchain { case watchsimulator case appletvos case appletvsimulator + case visionos = "xros" + case visionsimulator = "xrsimulator" case unknown } @@ -371,7 +406,8 @@ public final class DarwinToolchain: Toolchain { commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], frontendTargetInfo: FrontendTargetInfo, - driver: inout Driver + driver: inout Driver, + skipMacroOptions: Bool ) throws { guard let sdkPath = frontendTargetInfo.sdkPath?.path, let sdkInfo = getTargetSDKInfo(sdkPath: sdkPath) else { return } @@ -394,12 +430,27 @@ public final class DarwinToolchain: Toolchain { // doesn't always match the macosx sdk version so the compiler may fail to find // the prebuilt module in the versioned sub-dir. if frontendTargetInfo.target.triple.isMacCatalyst { + let resourceDirPath = VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path) + let basePrebuiltModulesPath = resourceDirPath.appending(components: "macosx", "prebuilt-modules") + + // Ensure we pass a path that exists. This matches logic used in the Swift frontend. + let prebuiltModulesPath: VirtualPath = try { + var versionString = sdkInfo.versionString + repeat { + let versionedPrebuiltModulesPath = + basePrebuiltModulesPath.appending(component: versionString) + if try fileSystem.exists(versionedPrebuiltModulesPath) { + return versionedPrebuiltModulesPath + } else if versionString.hasSuffix(".0") { + versionString.removeLast(2) + } else { + return basePrebuiltModulesPath + } + } while true + }() + commandLine.appendFlag(.prebuiltModuleCachePath) - commandLine.appendPath(try getToolPath(.swiftCompiler).parentDirectory/*bin*/ - .parentDirectory/*usr*/ - .appending(component: "lib").appending(component: "swift") - .appending(component: "macosx").appending(component: "prebuilt-modules") - .appending(component: sdkInfo.versionString)) + commandLine.appendPath(prebuiltModulesPath) } // Pass down -clang-target. @@ -422,41 +473,73 @@ public final class DarwinToolchain: Toolchain { commandLine.appendFlag(.clangTarget) commandLine.appendFlag(clangTargetTriple) + + // Repeat the above for the '-target-variant' flag + if driver.parsedOptions.contains(.targetVariant), + driver.isFrontendArgSupported(.clangTargetVariant), + let targetVariantTripleStr = frontendTargetInfo.targetVariant?.triple { + let clangTargetVariantTriple: String + if let explicitClangTargetVariantArg = driver.parsedOptions.getLastArgument(.clangTargetVariant)?.asSingle { + clangTargetVariantTriple = explicitClangTargetVariantArg + } else { + let currentVariantTriple = targetVariantTripleStr + let sdkVersionedOSSString = currentVariantTriple.osNameUnversioned + sdkInfo.sdkVersion(for: currentVariantTriple).sdkVersionString + clangTargetVariantTriple = currentVariantTriple.triple.replacingOccurrences(of: currentVariantTriple.osName, with: sdkVersionedOSSString) + } + + commandLine.appendFlag(.clangTargetVariant) + commandLine.appendFlag(clangTargetVariantTriple) + } } - if driver.isFrontendArgSupported(.externalPluginPath) { - // Default paths for compiler plugins found within an SDK (accessed via - // that SDK's plugin server). - let sdkPathRoot = VirtualPath.lookup(sdkPath).appending(components: "usr") - commandLine.appendFlag(.externalPluginPath) - commandLine.appendFlag("\(sdkPathRoot.pluginPath.name)#\(sdkPathRoot.pluginServerPath.name)") - - commandLine.appendFlag(.externalPluginPath) - commandLine.appendFlag("\(sdkPathRoot.localPluginPath.name)#\(sdkPathRoot.pluginServerPath.name)") - - // Determine the platform path. For simulator platforms, look into the - // corresponding device platform instance. - let origPlatformPath = VirtualPath.lookup(sdkPath) - .parentDirectory - .parentDirectory - .parentDirectory - let platformPath: VirtualPath - if let simulatorRange = origPlatformPath.basename.range(of: "Simulator.platform") { - let devicePlatform = origPlatformPath.basename.replacingCharacters(in: simulatorRange, with: "OS.platform") - platformPath = origPlatformPath.parentDirectory.appending(component: devicePlatform) - } else { - platformPath = origPlatformPath + if driver.isFrontendArgSupported(.externalPluginPath) && !skipMacroOptions { + // If the PLATFORM_DIR environment variable is set, also add plugin + // paths into it. Since this is a user override, it comes beore the + // default platform path that's based on the SDK. + if let platformDir = env["PLATFORM_DIR"], + let platformPath = try? VirtualPath(path: platformDir) { + addPluginPaths( + forPlatform: platformPath, + commandLine: &commandLine + ) } - // Default paths for compiler plugins within the platform (accessed via that - // platform's plugin server). - let platformPathRoot = platformPath.appending(components: "Developer", "usr") - commandLine.appendFlag(.externalPluginPath) - commandLine.appendFlag("\(platformPathRoot.pluginPath.name)#\(platformPathRoot.pluginServerPath.name)") + // Determine the platform path based on the SDK path. + addPluginPaths( + forPlatform: VirtualPath.lookup(sdkPath) + .parentDirectory + .parentDirectory + .parentDirectory, + commandLine: &commandLine + ) + } + } - commandLine.appendFlag(.externalPluginPath) - commandLine.appendFlag("\(platformPathRoot.localPluginPath.name)#\(platformPathRoot.pluginServerPath.name)") + /// Given the platform path (e.g., a path into something like iPhoneOS.platform), + /// add external plugin path arguments for compiler plugins that are distributed + /// within that path. + func addPluginPaths( + forPlatform origPlatformPath: VirtualPath, + commandLine: inout [Job.ArgTemplate] + ) { + // For simulator platforms, look into the corresponding device platform instance, + // because they share compiler plugins. + let platformPath: VirtualPath + if let simulatorRange = origPlatformPath.basename.range(of: "Simulator.platform") { + let devicePlatform = origPlatformPath.basename.replacingCharacters(in: simulatorRange, with: "OS.platform") + platformPath = origPlatformPath.parentDirectory.appending(component: devicePlatform) + } else { + platformPath = origPlatformPath } + + // Default paths for compiler plugins within the platform (accessed via that + // platform's plugin server). + let platformPathRoot = platformPath.appending(components: "Developer", "usr") + commandLine.appendFlag(.externalPluginPath) + commandLine.appendFlag("\(platformPathRoot.pluginPath.name)#\(platformPathRoot.pluginServerPath.name)") + + commandLine.appendFlag(.externalPluginPath) + commandLine.appendFlag("\(platformPathRoot.localPluginPath.name)#\(platformPathRoot.pluginServerPath.name)") } } diff --git a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift index 7abe9d771..89fa2bca1 100644 --- a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift @@ -14,6 +14,34 @@ import protocol TSCBasic.FileSystem import struct TSCBasic.AbsolutePath import var TSCBasic.localFileSystem +internal enum AndroidNDK { + internal static func getOSName() -> String? { + // The NDK is only available on macOS, linux and windows hosts currently. +#if os(Windows) + "windows" +#elseif os(Linux) + "linux" +#elseif os(macOS) + "darwin" +#else + nil +#endif + } + + internal static func getDefaultSysrootPath(in env: [String:String]) -> AbsolutePath? { + // The NDK is only available on an x86_64 hosts currently. +#if arch(x86_64) + guard let ndk = env["ANDROID_NDK_ROOT"], let os = getOSName() else { return nil } + return try? AbsolutePath(validating: ndk) + .appending(components: "toolchains", "llvm", "prebuilt") + .appending(component: "\(os)-x86_64") + .appending(component: "sysroot") +#else + return nil +#endif + } +} + /// Toolchain for Unix-like systems. public final class GenericUnixToolchain: Toolchain { public let env: [String: String] @@ -50,6 +78,12 @@ public final class GenericUnixToolchain: Toolchain { } } + public func addAutoLinkFlags(for linkLibraries: [LinkLibraryInfo], to commandLine: inout [Job.ArgTemplate]) { + for linkLibrary in linkLibraries { + commandLine.appendFlag("-l\(linkLibrary.linkName)") + } + } + /// Retrieve the absolute path for a given tool. public func getToolPath(_ tool: Tool) throws -> AbsolutePath { // Check the cache @@ -117,4 +151,33 @@ public final class GenericUnixToolchain: Toolchain { let environment = (targetTriple.environment == .android) ? "-android" : "" return "libclang_rt.\(sanitizer.libraryName)-\(targetTriple.archName)\(environment).a" } + + public func addPlatformSpecificCommonFrontendOptions( + commandLine: inout [Job.ArgTemplate], + inputs: inout [TypedVirtualPath], + frontendTargetInfo: FrontendTargetInfo, + driver: inout Driver, + skipMacroOptions: Bool + ) throws { + if let sysroot = driver.parsedOptions.getLastArgument(.sysroot)?.asSingle { + commandLine.appendFlag("-sysroot") + try commandLine.appendPath(VirtualPath(path: sysroot)) + } else if driver.targetTriple.environment == .android, + let sysroot = AndroidNDK.getDefaultSysrootPath(in: self.env) + { + commandLine.appendFlag("-sysroot") + try commandLine.appendPath(VirtualPath(path: sysroot.pathString)) + } + + if driver.targetTriple.os == .openbsd && driver.targetTriple.arch == .aarch64 { + if frontendTargetInfo.target.openbsdBTCFIEnabled ?? false { + commandLine.appendFlag(.Xcc) + commandLine.appendFlag("-Xclang=-msign-return-address=non-leaf") + commandLine.appendFlag(.Xcc) + commandLine.appendFlag("-Xclang=-msign-return-address-key=a_key") + commandLine.appendFlag(.Xcc) + commandLine.appendFlag("-Xclang=-mbranch-target-enforce") + } + } + } } diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index c5fdad5c8..c696ca0be 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -115,6 +115,9 @@ public protocol Toolchain { /// Constructs a proper output file name for a linker product. func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String + /// Adds linker flags corresponding to the specified set of link libraries + func addAutoLinkFlags(for linkLibraries: [LinkLibraryInfo], to commandLine: inout [Job.ArgTemplate]) + /// Perform platform-specific argument validation. func validateArgs(_ parsedOptions: inout ParsedOptions, targetTriple: Triple, @@ -122,6 +125,9 @@ public protocol Toolchain { compilerOutputType: FileType?, diagnosticsEngine: DiagnosticsEngine) throws + /// Return the DWARF version to emit, in the absence of arguments to the contrary. + func getDefaultDwarfVersion(targetTriple: Triple) -> UInt8 + /// Adds platform-specific linker flags to the provided command line func addPlatformSpecificLinkerArgs( to commandLine: inout [Job.ArgTemplate], @@ -151,7 +157,8 @@ public protocol Toolchain { commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], frontendTargetInfo: FrontendTargetInfo, - driver: inout Driver + driver: inout Driver, + skipMacroOptions: Bool ) throws var dummyForTestingObjectFormat: Triple.ObjectFormat {get} @@ -193,7 +200,10 @@ extension Toolchain { /// - Returns: String in the form of: `SWIFT_DRIVER_TOOLNAME_EXEC` private func envVarName(for toolName: String) -> String { - let lookupName = toolName.replacingOccurrences(of: "-", with: "_").uppercased() + let lookupName = toolName + .replacingOccurrences(of: "-", with: "_") + .replacingOccurrences(of: "+", with: "X") + .uppercased() return "SWIFT_DRIVER_\(lookupName)_EXEC" } @@ -216,14 +226,18 @@ extension Toolchain { let path = lookupExecutablePath(filename: executableName(executable), currentWorkingDirectory: nil, searchPaths: [toolDir]) { // Looking for tools from the tools directory. return path - } else if let path = lookupExecutablePath(filename: executableName(executable), currentWorkingDirectory: nil, searchPaths: [try executableDir]) { + } else if let path = lookupExecutablePath(filename: executableName(executable), currentWorkingDirectory: fileSystem.currentWorkingDirectory, searchPaths: [try executableDir]) { return path - } else if let path = try? xcrunFind(executable: executableName(executable)) { + } +#if canImport(Darwin) + if let path = try? xcrunFind(executable: executableName(executable)) { return path - } else if !["swift-frontend", "swift"].contains(executable), - let parentDirectory = try? getToolPath(.swiftCompiler).parentDirectory, - try parentDirectory != executableDir, - let path = lookupExecutablePath(filename: executableName(executable), searchPaths: [parentDirectory]) { + } +#endif + if !["swift-frontend", "swift"].contains(executable), + let parentDirectory = try? getToolPath(.swiftCompiler).parentDirectory, + try parentDirectory != executableDir, + let path = lookupExecutablePath(filename: executableName(executable), searchPaths: [parentDirectory]) { // If the driver library's client and the frontend are in different directories, // try looking for tools next to the frontend. return path @@ -240,15 +254,19 @@ extension Toolchain { } else { return try AbsolutePath(validating: "/usr/bin/" + executable) } - } else { - throw ToolchainError.unableToFind(tool: executable) } + + throw ToolchainError.unableToFind(tool: executable) } /// Looks for the executable in the `SWIFT_DRIVER_SWIFTSCAN_LIB` environment variable, if found nothing, /// looks in the `lib` relative to the compiler executable. /// TODO: If the driver needs to lookup other shared libraries, this is simple to generalize @_spi(Testing) public func lookupSwiftScanLib() throws -> AbsolutePath? { + if let overrideString = env["SWIFT_DRIVER_SWIFTSCAN_LIB"], + let path = try? AbsolutePath(validating: overrideString) { + return path + } #if os(Windows) // no matter if we are in a build tree or an installed tree, the layout is // always: `bin/_InternalSwiftScan.dll` @@ -256,27 +274,22 @@ extension Toolchain { .appending(component: "_InternalSwiftScan.dll") #else let libraryName = sharedLibraryName("lib_InternalSwiftScan") - if let overrideString = env["SWIFT_DRIVER_SWIFTSCAN_LIB"], - let path = try? AbsolutePath(validating: overrideString) { - return path - } else { - let compilerPath = try getToolPath(.swiftCompiler) - let toolchainRootPath = compilerPath.parentDirectory // bin - .parentDirectory // toolchain root - - let searchPaths = [toolchainRootPath.appending(component: "lib") - .appending(component: "swift") - .appending(component: compilerHostSupportLibraryOSComponent), - toolchainRootPath.appending(component: "lib") - .appending(component: "swift") - .appending(component: "host"), - // In case we are using a compiler from the build dir, we should also try - // this path. - toolchainRootPath.appending(component: "lib")] - for libraryPath in searchPaths.map({ $0.appending(component: libraryName) }) { - if fileSystem.isFile(libraryPath) { - return libraryPath - } + let compilerPath = try getToolPath(.swiftCompiler) + let toolchainRootPath = compilerPath.parentDirectory // bin + .parentDirectory // toolchain root + + let searchPaths = [toolchainRootPath.appending(component: "lib") + .appending(component: "swift") + .appending(component: compilerHostSupportLibraryOSComponent), + toolchainRootPath.appending(component: "lib") + .appending(component: "swift") + .appending(component: "host"), + // In case we are using a compiler from the build dir, we should also try + // this path. + toolchainRootPath.appending(component: "lib")] + for libraryPath in searchPaths.map({ $0.appending(component: libraryName) }) { + if fileSystem.isFile(libraryPath) { + return libraryPath } } @@ -284,33 +297,13 @@ extension Toolchain { #endif } - /// Looks for the executable in the `SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB` environment variable, if found nothing, - /// looks in the `lib` relative to the compiler executable. + /// Looks for the executable in the `SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB` environment variable. @_spi(Testing) public func lookupToolchainCASPluginLib() throws -> AbsolutePath? { if let overrideString = env["SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB"], let path = try? AbsolutePath(validating: overrideString) { return path } -#if os(Windows) return nil -#else - // Try to look for libToolchainCASPlugin in the developer dir, if found, - // prefer using that. Otherwise, just return nil, and auto fallback to - // builtin CAS. - let libraryName = sharedLibraryName("libToolchainCASPlugin") - let compilerPath = try getToolPath(.swiftCompiler) - let developerPath = compilerPath.parentDirectory // bin - .parentDirectory // toolchain root - .parentDirectory // toolchains - .parentDirectory // developer - let libraryPath = developerPath.appending(component: "usr") - .appending(component: "lib") - .appending(component: libraryName) - if fileSystem.isFile(libraryPath) { - return libraryPath - } - return nil -#endif } private func xcrunFind(executable: String) throws -> AbsolutePath { @@ -331,11 +324,14 @@ extension Toolchain { compilerOutputType: FileType?, diagnosticsEngine: DiagnosticsEngine) {} + public func getDefaultDwarfVersion(targetTriple: Triple) -> UInt8 { return 4 } + public func addPlatformSpecificCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], frontendTargetInfo: FrontendTargetInfo, - driver: inout Driver + driver: inout Driver, + skipMacroOptions: Bool ) throws {} /// Resolves the path to the given tool and whether or not it supports response files so that it diff --git a/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift b/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift index b842da9f1..86748c240 100644 --- a/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift @@ -72,13 +72,19 @@ public final class WebAssemblyToolchain: Toolchain { case .executable: return moduleName case .dynamicLibrary: - // WASM doesn't support dynamic libraries yet, but we'll report the error later. + // Wasm doesn't support dynamic libraries yet, but we'll report the error later. return "" case .staticLibrary: return "lib\(moduleName).a" } } + public func addAutoLinkFlags(for linkLibraries: [LinkLibraryInfo], to commandLine: inout [Job.ArgTemplate]) { + for linkLibrary in linkLibraries { + commandLine.appendFlag("-l\(linkLibrary.linkName)") + } + } + /// Retrieve the absolute path for a given tool. public func getToolPath(_ tool: Tool) throws -> AbsolutePath { // Check the cache @@ -95,10 +101,7 @@ public final class WebAssemblyToolchain: Toolchain { switch tool { case .swiftCompiler: return try lookup(executable: "swift-frontend") - case .staticLinker(nil): - return try lookup(executable: "ar") - case .staticLinker(.llvmFull), - .staticLinker(.llvmThin): + case .staticLinker: return try lookup(executable: "llvm-ar") case .dynamicLinker: // FIXME: This needs to look in the tools_directory first. @@ -143,7 +146,12 @@ public final class WebAssemblyToolchain: Toolchain { targetTriple: Triple, isShared: Bool ) throws -> String { - throw Error.sanitizersUnsupportedForTarget(targetTriple.triple) + switch sanitizer { + case .address: + return "libclang_rt.\(sanitizer.libraryName)-\(targetTriple.archName).a" + default: + throw Error.sanitizersUnsupportedForTarget(targetTriple.triple) + } } public func platformSpecificInterpreterEnvironmentVariables(env: [String : String], diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index 1d529ded6..de4d92572 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -113,6 +113,12 @@ extension WindowsToolchain.ToolchainValidationError { } } + public func addAutoLinkFlags(for linkLibraries: [LinkLibraryInfo], to commandLine: inout [Job.ArgTemplate]) { + for linkLibrary in linkLibraries { + commandLine.appendFlag("-l\(linkLibrary.linkName)") + } + } + public func defaultSDKPath(_ target: Triple?) throws -> AbsolutePath? { // TODO(compnerd): replicate the SPM processing of the SDKInfo.plist if let SDKROOT = env["SDKROOT"] { @@ -130,6 +136,11 @@ extension WindowsToolchain.ToolchainValidationError { public func runtimeLibraryName(for sanitizer: Sanitizer, targetTriple: Triple, isShared: Bool) throws -> String { // TODO(compnerd) handle shared linking + + // FIXME(compnerd) when should `clang_rt.ubsan_standalone_cxx` be used? + if sanitizer == .undefinedBehavior { + return "clang_rt.ubsan_standalone.lib" + } return "clang_rt.\(sanitizer.libraryName).lib" } diff --git a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift index 90b89f828..8196c47d8 100644 --- a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift +++ b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift @@ -20,20 +20,20 @@ import class TSCBasic.Process /// TODO: It would be nice if build planning did not involve an executor. /// We must hunt down all uses of an executor during planning and move /// relevant compiler functionality into libSwiftScan. -internal class SimpleExecutor: DriverExecutor { - let resolver: ArgsResolver +@_spi(Testing) public class SimpleExecutor: DriverExecutor { + public let resolver: ArgsResolver let fileSystem: FileSystem let env: [String: String] - init(resolver: ArgsResolver, fileSystem: FileSystem, env: [String: String]) { + public init(resolver: ArgsResolver, fileSystem: FileSystem, env: [String: String]) { self.resolver = resolver self.fileSystem = fileSystem self.env = env } - func execute(job: Job, - forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { + public func execute(job: Job, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult { let arguments: [String] = try resolver.resolveArgumentList(for: job, useResponseFiles: .heuristic) var childEnv = env @@ -42,17 +42,17 @@ internal class SimpleExecutor: DriverExecutor { return try process.waitUntilExit() } - func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, - numParallelJobs: Int, forceResponseFiles: Bool, - recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { + public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate, + numParallelJobs: Int, forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws { fatalError("Unsupported operation on current executor") } - func checkNonZeroExit(args: String..., environment: [String : String]) throws -> String { + public func checkNonZeroExit(args: String..., environment: [String : String]) throws -> String { try Process.checkNonZeroExit(arguments: args, environment: environment) } - func description(of job: Job, forceResponseFiles: Bool) throws -> String { + public func description(of job: Job, forceResponseFiles: Bool) throws -> String { let useResponseFiles : ResponseFileHandling = forceResponseFiles ? .forced : .heuristic let (args, usedResponseFile) = try resolver.resolveArgumentList(for: job, useResponseFiles: useResponseFiles) var result = args.map { $0.spm_shellEscaped() }.joined(separator: " ") diff --git a/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift b/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift index 1fac02d79..751865b14 100644 --- a/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift +++ b/Sources/SwiftDriver/ToolingInterface/ToolingUtil.swift @@ -14,27 +14,156 @@ import class TSCBasic.DiagnosticsEngine import struct TSCBasic.Diagnostic import class TSCBasic.ProcessSet import enum TSCBasic.ProcessEnv +import struct TSCBasic.ProcessEnvironmentBlock import var TSCBasic.localFileSystem +import struct TSCBasic.AbsolutePath import SwiftOptions +//typedef enum { +// SWIFTDRIVER_TOOLING_DIAGNOSTIC_ERROR = 0, +// SWIFTDRIVER_TOOLING_DIAGNOSTIC_WARNING = 1, +// SWIFTDRIVER_TOOLING_DIAGNOSTIC_REMARK = 2, +// SWIFTDRIVER_TOOLING_DIAGNOSTIC_NOTE = 3 +//} swiftdriver_tooling_diagnostic_kind; +public let SWIFTDRIVER_TOOLING_DIAGNOSTIC_ERROR: CInt = 0; +public let SWIFTDRIVER_TOOLING_DIAGNOSTIC_WARNING: CInt = 1; +public let SWIFTDRIVER_TOOLING_DIAGNOSTIC_REMARK: CInt = 2; +public let SWIFTDRIVER_TOOLING_DIAGNOSTIC_NOTE: CInt = 3; + +@_cdecl("swift_getSingleFrontendInvocationFromDriverArgumentsV2") +public func getSingleFrontendInvocationFromDriverArgumentsV2(driverPath: UnsafePointer, + argListCount: CInt, + argList: UnsafePointer?>, + action: @convention(c) (CInt, UnsafePointer?>) -> Bool, + diagnosticCallback: @convention(c) (CInt, UnsafePointer) -> Void, + forceNoOutputs: Bool = false) -> Bool { + return getSingleFrontendInvocationFromDriverArgumentsV3(driverPath: driverPath, argListCount: argListCount, + argList: argList, action: action, + diagnosticCallback: diagnosticCallback, + compilerIntegratedTooling: true) +} + +@_cdecl("swift_getSingleFrontendInvocationFromDriverArgumentsV3") +public func getSingleFrontendInvocationFromDriverArgumentsV3(driverPath: UnsafePointer, + argListCount: CInt, + argList: UnsafePointer?>, + action: @convention(c) (CInt, UnsafePointer?>) -> Bool, + diagnosticCallback: @convention(c) (CInt, UnsafePointer) -> Void, + compilerIntegratedTooling: Bool = false, + forceNoOutputs: Bool = false) -> Bool { + // Bridge the driver path argument + let bridgedDriverPath = String(cString: driverPath) + + // Bridge the argv equivalent + let argListBufferPtr = UnsafeBufferPointer?>(start: argList, count: Int(argListCount)) + let bridgedArgList = [bridgedDriverPath] + argListBufferPtr.map { String(cString: $0!) } + + // Bridge the action callback + let bridgedAction: ([String]) -> Bool = { args in + return withArrayOfCStrings(args) { + return action(CInt(args.count), $0!) + } + } + + // Bridge the diagnostic callback + let bridgedDiagnosticCallback: (CInt, String) -> Void = { diagKind, message in + diagnosticCallback(diagKind, message) + } + + var diagnostics: [Diagnostic] = [] + let result = getSingleFrontendInvocationFromDriverArgumentsV2(driverPath: bridgedDriverPath, + argList: bridgedArgList, + action: bridgedAction, + diagnostics: &diagnostics, + diagnosticCallback: bridgedDiagnosticCallback, + compilerIntegratedTooling: compilerIntegratedTooling, + forceNoOutputs: forceNoOutputs) + return result +} + +public func getSingleFrontendInvocationFromDriverArgumentsV2(driverPath: String, + argList: [String], + action: ([String]) -> Bool, + diagnostics: inout [Diagnostic], + diagnosticCallback: @escaping (CInt, String) -> Void, + compilerIntegratedTooling: Bool = false, + forceNoOutputs: Bool = false) -> Bool { + let env = ProcessEnv.vars + let executor: SimpleExecutor + do { + let resolver = try ArgsResolver(fileSystem: localFileSystem) + executor = SimpleExecutor(resolver: resolver, + fileSystem: localFileSystem, + env: env) + } catch { + print("Unexpected error: \(error)") + return true + } + + return getSingleFrontendInvocationFromDriverArgumentsV3(driverPath: driverPath, argList: argList, action: action, diagnostics: &diagnostics, diagnosticCallback: diagnosticCallback, env: env, executor: executor, compilerIntegratedTooling: compilerIntegratedTooling, forceNoOutputs: forceNoOutputs) +} + +public func getSingleFrontendInvocationFromDriverArgumentsV3(driverPath: String, + argList: [String], + action: ([String]) -> Bool, + diagnostics: inout [Diagnostic], + diagnosticCallback: @escaping (CInt, String) -> Void, + env: [String: String], + executor: some DriverExecutor, + compilerIntegratedTooling: Bool = false, + forceNoOutputs: Bool = false) -> Bool { + return getSingleFrontendInvocationFromDriverArgumentsV4(driverPath: driverPath, argList: argList, action: action, diagnostics: &diagnostics, diagnosticCallback: diagnosticCallback, env: env, executor: executor, compilerIntegratedTooling: compilerIntegratedTooling, compilerExecutableDir: nil, forceNoOutputs: forceNoOutputs) +} + /// Generates the list of arguments that would be passed to the compiler -/// frontend from the given driver arguments. +/// frontend from the given driver arguments, for a single-compiler-invocation +/// context. /// -/// \param ArgList The driver arguments (i.e. normal arguments for \c swiftc). -/// \param ForceNoOutputs If true, override the output mode to "-typecheck" and +/// \param driverPath the driver executable path +/// \param argList The driver arguments (i.e. normal arguments for \c swiftc). +/// \param diagnostics Contains the diagnostics emitted by the driver +/// \param action invokes a user-provided action on the resulting frontend invocation command +/// \param compilerIntegratedTooling If true, this code is executed in the context of a +/// Swift compiler image which contains symbols normally queried from a libSwiftScan instance. +/// \param forceNoOutputs If true, override the output mode to "-typecheck" and /// produce no outputs. For example, this disables "-emit-module" and "-c" and /// prevents the creation of temporary files. -/// \param outputFrontendArgs Contains the resulting frontend invocation command -/// \param emittedDiagnostics Contains the diagnostics emitted by the driver /// /// \returns true on error /// /// \note This function is not intended to create invocations which are /// suitable for use in REPL or immediate modes. -public func getSingleFrontendInvocationFromDriverArguments(argList: [String], - outputFrontendArgs: inout [String], - emittedDiagnostics: inout [Diagnostic], - forceNoOutputs: Bool = false) -> Bool { +public func getSingleFrontendInvocationFromDriverArgumentsV4(driverPath: String, + argList: [String], + action: ([String]) -> Bool, + diagnostics: inout [Diagnostic], + diagnosticCallback: @escaping (CInt, String) -> Void, + env: [String: String], + executor: some DriverExecutor, + compilerIntegratedTooling: Bool = false, + compilerExecutableDir: AbsolutePath? = nil, + forceNoOutputs: Bool = false) -> Bool { + /// Handler for emitting diagnostics to tooling clients. + let toolingDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in + let diagnosticKind: CInt + switch diagnostic.message.behavior { + case .error: + diagnosticKind = SWIFTDRIVER_TOOLING_DIAGNOSTIC_ERROR + case .warning: + diagnosticKind = SWIFTDRIVER_TOOLING_DIAGNOSTIC_WARNING + case .note: + diagnosticKind = SWIFTDRIVER_TOOLING_DIAGNOSTIC_NOTE + case .remark: + diagnosticKind = SWIFTDRIVER_TOOLING_DIAGNOSTIC_REMARK + default: + diagnosticKind = SWIFTDRIVER_TOOLING_DIAGNOSTIC_ERROR + } + diagnosticCallback(diagnosticKind, diagnostic.message.text) + } + let diagnosticsEngine = DiagnosticsEngine(handlers: [toolingDiagnosticsHandler]) + defer { diagnostics = diagnosticsEngine.diagnostics } + + var singleFrontendTaskCommand: [String] = [] var args: [String] = [] args.append(contentsOf: argList) @@ -55,13 +184,10 @@ public func getSingleFrontendInvocationFromDriverArguments(argList: [String], args.append("-driver-filelist-threshold"); args.append(String(Int.max)); - let diagnosticsEngine = DiagnosticsEngine() - defer { emittedDiagnostics = diagnosticsEngine.diagnostics } - do { - args = try ["swiftc"] + Driver.expandResponseFiles(args, - fileSystem: localFileSystem, - diagnosticsEngine: diagnosticsEngine) + args = try [driverPath] + Driver.expandResponseFiles(args, + fileSystem: localFileSystem, + diagnosticsEngine: diagnosticsEngine) let optionTable = OptionTable() var parsedOptions = try optionTable.parse(Array(args), for: .batch, delayThrows: true) @@ -73,18 +199,16 @@ public func getSingleFrontendInvocationFromDriverArguments(argList: [String], } // Instantiate the driver, setting up the toolchain in the process, etc. - let resolver = try ArgsResolver(fileSystem: localFileSystem) - let executor = SimpleExecutor(resolver: resolver, - fileSystem: localFileSystem, - env: ProcessEnv.vars) var driver = try Driver(args: parsedOptions.commandLine, + env: env, diagnosticsOutput: .engine(diagnosticsEngine), - executor: executor) + executor: executor, + compilerIntegratedTooling: compilerIntegratedTooling, + compilerExecutableDir: compilerExecutableDir) if diagnosticsEngine.hasErrors { return true } - let buildPlan = try driver.planBuild() if diagnosticsEngine.hasErrors { return true @@ -98,12 +222,11 @@ public func getSingleFrontendInvocationFromDriverArguments(argList: [String], diagnosticsEngine.emit(.error_expected_frontend_command()) return true } - outputFrontendArgs = try executor.description(of: compileJob, - forceResponseFiles: false).components(separatedBy: " ") + singleFrontendTaskCommand = try executor.resolver.resolveArgumentList(for: compileJob, useResponseFiles: .disabled) } catch { print("Unexpected error: \(error).") return true } - return false + return action(singleFrontendTaskCommand) } diff --git a/Sources/SwiftDriver/Utilities/DOTModuleDependencyGraphSerializer.swift b/Sources/SwiftDriver/Utilities/DOTModuleDependencyGraphSerializer.swift index 602bbe944..ba89b1689 100644 --- a/Sources/SwiftDriver/Utilities/DOTModuleDependencyGraphSerializer.swift +++ b/Sources/SwiftDriver/Utilities/DOTModuleDependencyGraphSerializer.swift @@ -64,10 +64,7 @@ import TSCBasic stream.write("digraph Modules {\n") for (moduleId, moduleInfo) in graph.modules { stream.write(outputNode(for: moduleId)) - guard let dependencies = moduleInfo.directDependencies else { - continue - } - for dependencyId in dependencies { + for dependencyId in moduleInfo.allDependencies { stream.write(" \(quoteName(label(for: moduleId))) -> \(quoteName(label(for: dependencyId))) [color=black];\n") } } diff --git a/Sources/SwiftDriver/Utilities/DateAdditions.swift b/Sources/SwiftDriver/Utilities/DateAdditions.swift index fcad533de..589df0fe2 100644 --- a/Sources/SwiftDriver/Utilities/DateAdditions.swift +++ b/Sources/SwiftDriver/Utilities/DateAdditions.swift @@ -12,12 +12,14 @@ #if os(Windows) import WinSDK -#elseif os(iOS) || os(macOS) || os(tvOS) || os(watchOS) +#elseif canImport(Darwin) import Darwin #elseif canImport(Glibc) import Glibc #elseif canImport(Musl) import Musl +#elseif canImport(Bionic) +import Bionic #endif /// Represents a time point value with nanosecond precision. diff --git a/Sources/SwiftDriver/Utilities/Diagnostics.swift b/Sources/SwiftDriver/Utilities/Diagnostics.swift index a7ceb422d..8b41e2d05 100644 --- a/Sources/SwiftDriver/Utilities/Diagnostics.swift +++ b/Sources/SwiftDriver/Utilities/Diagnostics.swift @@ -35,6 +35,10 @@ extension Diagnostic.Message { .error("unsupported argument '\(argument)' to option '\(option.spelling)'") } + static func error_unsupported_opt_for_frontend(option: Option) -> Diagnostic.Message { + .error("frontend does not support option '\(option.spelling)'") + } + static func error_option_requires_sanitizer(option: Option) -> Diagnostic.Message { .error("option '\(option.spelling)' requires a sanitizer to be enabled. Use -sanitize= to enable a sanitizer") } @@ -51,12 +55,12 @@ extension Diagnostic.Message { .warning("inferring simulator environment for target '\(originalTriple.triple)'; use '-target \(inferredTriple.triple)' instead") } - static func warning_inprocess_target_info_query_failed(_ error: String) -> Diagnostic.Message { - .warning("In-process target-info query failed (\(error)). Using fallback mechanism.") + static func remark_inprocess_target_info_query_failed(_ error: String) -> Diagnostic.Message { + .remark("In-process target-info query failed (\(error)). Using fallback mechanism.") } - static func warning_inprocess_supported_features_query_failed(_ error: String) -> Diagnostic.Message { - .warning("In-process supported-compiler-features query failed (\(error)). Using fallback mechanism.") + static func remark_inprocess_supported_features_query_failed(_ error: String) -> Diagnostic.Message { + .remark("In-process supported-compiler-features query failed (\(error)). Using fallback mechanism.") } static func error_argument_not_allowed_with(arg: String, other: String) -> Diagnostic.Message { @@ -166,4 +170,16 @@ extension Diagnostic.Message { static func error_expected_frontend_command() -> Diagnostic.Message { .error("expected a swift frontend command") } + + static var error_no_library_evolution_embedded: Diagnostic.Message { + .error("Library evolution cannot be enabled with embedded Swift.") + } + + static var error_need_wmo_embedded: Diagnostic.Message { + .error("Whole module optimization (wmo) must be enabled with embedded Swift.") + } + + static var error_no_objc_interop_embedded: Diagnostic.Message { + .error("Objective-C interop cannot be enabled with embedded Swift.") + } } diff --git a/Sources/SwiftDriver/Utilities/FileType.swift b/Sources/SwiftDriver/Utilities/FileType.swift index 72076d9b9..67d5f42c6 100644 --- a/Sources/SwiftDriver/Utilities/FileType.swift +++ b/Sources/SwiftDriver/Utilities/FileType.swift @@ -54,6 +54,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// An SPI Swift Interface file. case privateSwiftInterface = "private.swiftinterface" + /// An interface file containng package decls as well as SPI and public decls. + case packageSwiftInterface = "package.swiftinterface" + /// Serialized source information. case swiftSourceInfoFile = "swiftsourceinfo" @@ -69,6 +72,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// Raw sib file case raw_sib + /// Raw LLVM IR file + case raw_llvmIr + /// LLVM IR file case llvmIR = "ll" @@ -111,6 +117,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// JSON-based -emit-supported-features output case jsonCompilerFeatures = "compilerFeatures.json" + /// JSON-based -print-supported-features output + case jsonSupportedFeatures = "supportedFeatures.json" + /// JSON-based binary Swift module artifact description case jsonSwiftArtifacts = "artifacts.json" @@ -151,6 +160,18 @@ public enum FileType: String, Hashable, CaseIterable, Codable { /// ABI baseline JSON case jsonABIBaseline = "abi.json" + + /// API descriptor JSON + case jsonAPIDescriptor + + /// Swift Module Summary + case moduleSummary = "swiftmodulesummary" + + /// Swift Module Semantic Info + case moduleSemanticInfo + + /// Cached Diagnostics + case cachedDiagnostics } extension FileType: CustomStringConvertible { @@ -173,6 +194,9 @@ extension FileType: CustomStringConvertible { case .raw_sib: return "raw-sib" + case .raw_llvmIr: + return "raw-llvm-ir" + case .llvmIR: return "llvm-ir" @@ -182,6 +206,9 @@ extension FileType: CustomStringConvertible { case .privateSwiftInterface: return "private-swiftinterface" + case .packageSwiftInterface: + return "package-swiftinterface" + case .objcHeader: return "objc-header" @@ -235,6 +262,21 @@ extension FileType: CustomStringConvertible { case .swiftConstValues: return "const-values" + + case .jsonAPIDescriptor: + return "api-descriptor-json" + + case .moduleSummary: + return "swift-module-summary" + + case .moduleSemanticInfo: + return "module-semantic-info" + + case .cachedDiagnostics: + return "cached-diagnostics" + + case .jsonSupportedFeatures: + return "json-supported-swift-features" } } } @@ -251,10 +293,12 @@ extension FileType { .emitModuleDependencies, .swiftDocumentation, .pcm, .diagnostics, .emitModuleDiagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord, - .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, + .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .jsonDependencies, .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts, .indexUnitOutputPath, .modDepCache, .jsonAPIBaseline, - .jsonABIBaseline, .swiftConstValues: + .jsonABIBaseline, .swiftConstValues, .jsonAPIDescriptor, + .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr, + .jsonSupportedFeatures: return false } } @@ -299,6 +343,8 @@ extension FileType { return "swiftinterface" case .privateSwiftInterface: return "private-swiftinterface" + case .packageSwiftInterface: + return "package-swiftinterface" case .swiftSourceInfoFile: return "swiftsourceinfo" case .clangModuleMap: @@ -319,6 +365,8 @@ extension FileType { return "raw-sil" case .raw_sib: return "raw-sib" + case .raw_llvmIr: + return "raw-llvm-ir" case .llvmIR: return "llvm-ir" case .llvmBitcode: @@ -359,6 +407,16 @@ extension FileType { return "abi-baseline-json" case .swiftConstValues: return "const-values" + case .jsonAPIDescriptor: + return "api-descriptor-json" + case .moduleSummary: + return "swiftmodulesummary" + case .moduleSemanticInfo: + return "module-semantic-info" + case .cachedDiagnostics: + return "cached-diagnostics" + case .jsonSupportedFeatures: + return "json-supported-swift-features" } } } @@ -368,9 +426,11 @@ extension FileType { switch self { case .swift, .sil, .dependencies, .emitModuleDependencies, .assembly, .ast, .raw_sil, .llvmIR,.objcHeader, .autolink, .importedModules, .tbd, - .moduleTrace, .yamlOptimizationRecord, .swiftInterface, .privateSwiftInterface, + .moduleTrace, .yamlOptimizationRecord, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, - .jsonSwiftArtifacts, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues: + .jsonSwiftArtifacts, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues, + .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, + .raw_llvmIr, .jsonSupportedFeatures: return true case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule, .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics, @@ -388,13 +448,47 @@ extension FileType { return true case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .emitModuleDependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, - .privateSwiftInterface, .swiftSourceInfoFile, .raw_sil, .raw_sib, + .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .emitModuleDiagnostics, .objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache, .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, - .indexUnitOutputPath, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues: + .indexUnitOutputPath, .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues, + .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, + .raw_llvmIr, .jsonSupportedFeatures: return false } } + + /// Returns true if producing the file type requires running SILGen. + var requiresSILGen: Bool { + switch self { + case .swift, .ast, .indexData, .indexUnitOutputPath, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSupportedFeatures: + return false + case .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink, .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, .swiftSourceInfoFile, .swiftConstValues, .assembly, .raw_sil, .raw_sib, .llvmIR, .llvmBitcode, .diagnostics, .emitModuleDiagnostics, .emitModuleDependencies, .objcHeader, .swiftDeps, .modDepCache, .remap, .importedModules, .tbd, .jsonDependencies, .jsonSwiftArtifacts, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, .pch, .clangModuleMap, .jsonAPIBaseline, .jsonABIBaseline, .jsonAPIDescriptor, .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr: + return true + } + } + + /// Returns true if the type can be cached as output. + var supportCaching: Bool { + switch self { + case .diagnostics, .emitModuleDiagnostics, // diagnostics are cached using cached diagnostics. + // Those are by-product from swift-driver and not considered outputs need caching. + .jsonSwiftArtifacts, .remap, .indexUnitOutputPath, .modDepCache, + // the remaining should not be an output from a caching swift job. + .swift, .image, .dSYM, .importedModules, .clangModuleMap, + .jsonCompilerFeatures, .jsonTargetInfo, .autolink, .jsonSupportedFeatures: + return false + case .assembly, .llvmIR, .llvmBitcode, .object, .sil, .sib, .ast, + .dependencies, .emitModuleDependencies, .swiftModule, + .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .packageSwiftInterface, + .swiftSourceInfoFile, .raw_sil, .raw_sib, .objcHeader, .swiftDeps, .tbd, + .moduleTrace, .indexData, .yamlOptimizationRecord, + .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, + .jsonAPIBaseline, .jsonABIBaseline, .swiftConstValues, .jsonAPIDescriptor, + .moduleSummary, .moduleSemanticInfo, .cachedDiagnostics, .raw_llvmIr: + return true + } + } } diff --git a/Sources/SwiftDriver/Utilities/Sanitizer.swift b/Sources/SwiftDriver/Utilities/Sanitizer.swift index cd566b812..33f84c832 100644 --- a/Sources/SwiftDriver/Utilities/Sanitizer.swift +++ b/Sources/SwiftDriver/Utilities/Sanitizer.swift @@ -16,6 +16,9 @@ public enum Sanitizer: String, Hashable { /// Address sanitizer (ASan) case address + // Address sanitizer Stable ABI (ASan) + case address_stable_abi + /// Thread sanitizer (TSan) case thread @@ -34,6 +37,7 @@ public enum Sanitizer: String, Hashable { var libraryName: String { switch self { case .address: return "asan" + case .address_stable_abi: return "asan_abi" case .thread: return "tsan" case .undefinedBehavior: return "ubsan" case .fuzzer: return "fuzzer" diff --git a/Sources/SwiftDriver/Utilities/StringAdditions.swift b/Sources/SwiftDriver/Utilities/StringAdditions.swift index 9feabd5b1..c84bd982a 100644 --- a/Sources/SwiftDriver/Utilities/StringAdditions.swift +++ b/Sources/SwiftDriver/Utilities/StringAdditions.swift @@ -21,6 +21,15 @@ extension String { return start.isValidSwiftIdentifierStart && continuation.allSatisfy { $0.isValidSwiftIdentifierContinuation } } + + /// Whether this string is a valid Swift raw identifier. + public var sd_isValidAsRawIdentifier: Bool { + guard !isEmpty else { + return false + } + return unicodeScalars.allSatisfy { $0.isValidRawIdentifierScalar } + && !unicodeScalars.allSatisfy { $0.isPermittedRawIdentifierWhitespace } + } } extension DefaultStringInterpolation { @@ -120,6 +129,43 @@ extension Unicode.Scalar { /// `true` if this character is an ASCII digit: [0-9] var isASCIIDigit: Bool { (0x30...0x39).contains(value) } + /// `true` if this scalar is valid in a Swift raw identifier. + var isValidRawIdentifierScalar: Bool { + // A raw identifier is terminated by a backtick, and the backslash is reserved for possible + // future escaping. + if self == "`" || self == "\\" { + return false + } + // Unprintable ASCII control characters are forbidden. + if (0x0000...0x001F).contains(value) || value == 0x007F { + return false; + } + return !isForbiddenRawIdentifierWhitespace + } + + var isForbiddenRawIdentifierWhitespace: Bool { + // This is the set of code points satisfying the `White_Space` property, excluding the set + // satisfying the `Pattern_White_Space` property, and excluding any other ASCII non-printables + // and Unicode separators. In other words, the only whitespace code points allowed in a raw + // identifier are U+0020, and U+200E/200F (LTR/RTL marks) (see + // `isPermittedRawIdentifierWhitespace` below). + return (0x0009...0x000D).contains(value) || + value == 0x0085 || + value == 0x00A0 || + value == 0x1680 || + (0x2000...0x200A).contains(value) || + (0x2028...0x2029).contains(value) || + value == 0x202F || + value == 0x205F || + value == 0x3000 + } + + var isPermittedRawIdentifierWhitespace: Bool { + return value == 0x0020 || + value == 0x200E || + value == 0x200F + } + /// `true` if this is a body character of a C identifier, /// which is [a-zA-Z0-9_]. func isCIdentifierBody(allowDollar: Bool = false) -> Bool { diff --git a/Sources/SwiftDriver/Utilities/System.swift b/Sources/SwiftDriver/Utilities/System.swift index c4fdaa89d..8c25e831e 100644 --- a/Sources/SwiftDriver/Utilities/System.swift +++ b/Sources/SwiftDriver/Utilities/System.swift @@ -16,6 +16,8 @@ import Darwin import Glibc #elseif canImport(Musl) import Musl +#elseif canImport(Android) +import Android #endif func argumentNeedsQuoting(_ argument: String) -> Bool { @@ -57,7 +59,7 @@ func quoteArgument(_ argument: String) -> String { #endif } -#if canImport(Darwin) || os(Linux) || os(Android) || os(OpenBSD) +#if canImport(Darwin) || os(Linux) || os(Android) || os(OpenBSD) || os(FreeBSD) // Adapted from llvm::sys::commandLineFitsWithinSystemLimits. func commandLineFitsWithinSystemLimits(path: String, args: [String]) -> Bool { let upperBound = sysconf(Int32(_SC_ARG_MAX)) diff --git a/Sources/SwiftDriver/Utilities/Triple+Platforms.swift b/Sources/SwiftDriver/Utilities/Triple+Platforms.swift index 380560093..d66c17314 100644 --- a/Sources/SwiftDriver/Utilities/Triple+Platforms.swift +++ b/Sources/SwiftDriver/Utilities/Triple+Platforms.swift @@ -32,6 +32,9 @@ public enum DarwinPlatform: Hashable { /// watchOS, corresponding to the `watchos` OS name. case watchOS(EnvironmentWithoutCatalyst) + /// visionOS, corresponding to the `visionos` OS name. + case visionOS(EnvironmentWithoutCatalyst) + /// The most general form of environment information attached to a /// `DarwinPlatform`. /// @@ -76,6 +79,9 @@ public enum DarwinPlatform: Hashable { case .watchOS: guard let withoutCatalyst = environment.withoutCatalyst else { return nil } return .watchOS(withoutCatalyst) + case .visionOS: + guard let withoutCatalyst = environment.withoutCatalyst else { return nil } + return .visionOS(withoutCatalyst) } } @@ -97,6 +103,10 @@ public enum DarwinPlatform: Hashable { return "watchOS" case .watchOS(.simulator): return "watchOS Simulator" + case .visionOS(.device): + return "visionOS" + case .visionOS(.simulator): + return "visionOS Simulator" } } @@ -120,6 +130,10 @@ public enum DarwinPlatform: Hashable { return "watchos" case .watchOS(.simulator): return "watchsimulator" + case .visionOS(.device): + return "xros" + case .visionOS(.simulator): + return "xrsimulator" } } @@ -142,6 +156,10 @@ public enum DarwinPlatform: Hashable { return "watchos" case .watchOS(.simulator): return "watchos-simulator" + case .visionOS(.device): + return "xros" + case .visionOS(.simulator): + return "xros-simulator" } } @@ -165,6 +183,10 @@ public enum DarwinPlatform: Hashable { return "watchos" case .watchOS(.simulator): return "watchossim" + case .visionOS(.device): + return "xros" + case .visionOS(.simulator): + return "xrossim" } } } @@ -197,6 +219,8 @@ extension Triple { return _iOSVersion case .watchOS: return _watchOSVersion + case .visionOS: + return _visionOSVersion } } @@ -223,6 +247,8 @@ extension Triple { return .watchOS(makeEnvironment()) case .tvos: return .tvOS(makeEnvironment()) + case .visionos: + return .visionOS(makeEnvironment()) default: return nil } @@ -272,10 +298,33 @@ extension Triple { } return osVersion + case .visionOS(_): + return _visionOSVersion + } + } + + + /// The "os" component of the Clang compiler resource library directory (`/lib/`). + /// Must be kept in sync with Clang driver: + /// https://github.com/llvm/llvm-project/blob/llvmorg-20.1.4/clang/lib/Driver/ToolChain.cpp#L690 + @_spi(Testing) public var clangOSLibName: String { + guard let os else { + return osName + } + if os.isDarwin { + return "darwin" + } + + switch os { + case .freeBSD: return "freebsd" + case .netbsd: return "netbsd" + case .openbsd: return "openbsd" + case .aix: return "aix" + default: return osName } } - /// The platform name, i.e. the name clang uses to identify this target in its + /// The platform name, i.e. the name Swift uses to identify this target in its /// resource directory. /// /// - Parameter conflatingDarwin: If true, all Darwin platforms will be @@ -285,14 +334,25 @@ extension Triple { switch os { case nil: fatalError("unknown OS") - case .darwin, .macosx, .ios, .tvos, .watchos: + case .darwin, .macosx, .ios, .tvos, .watchos, .visionos: guard let darwinPlatform = darwinPlatform else { fatalError("unsupported darwin platform kind?") } return conflatingDarwin ? "darwin" : darwinPlatform.platformName case .linux: - return environment == .android ? "android" : "linux" + switch environment { + case .musl where vendor == .swift: + // The triple for linux-static is -swift-linux-musl, to distinguish + // it from a "normal" musl set-up (ala Alpine). + return "linux-static" + case .musl, .musleabihf, .musleabi: + return "musl" + case .android: + return "android" + default: + return "linux" + } case .freeBSD: return "freebsd" case .openbsd: @@ -335,9 +395,9 @@ extension Triple { /// `tripleVersion >= featureVersion`. /// /// - SeeAlso: `Triple.supports(_:)` - public struct FeatureAvailability { + public struct FeatureAvailability: Sendable { - public enum Availability { + public enum Availability: Sendable { case unavailable case available(since: Version) case availableInAllVersions @@ -347,6 +407,7 @@ extension Triple { public let iOS: Availability public let tvOS: Availability public let watchOS: Availability + public var visionOS: Availability // TODO: We should have linux, windows, etc. public let nonDarwin: Bool @@ -365,8 +426,14 @@ extension Triple { self.tvOS = tvOS self.watchOS = watchOS self.nonDarwin = nonDarwin + self.visionOS = iOS } + public func withVisionOS(_ visionOS: Availability) -> FeatureAvailability { + var res = self + res.visionOS = visionOS + return res + } /// Returns the version when the feature was introduced on the specified Darwin /// platform, or `.unavailable` if the feature has not been introduced there. public subscript(darwinPlatform: DarwinPlatform) -> Availability { @@ -379,6 +446,8 @@ extension Triple { return tvOS case .watchOS: return watchOS + case .visionOS: + return visionOS } } } diff --git a/Sources/SwiftDriver/Utilities/Triple.swift b/Sources/SwiftDriver/Utilities/Triple.swift index e3cc71dbb..714babc4b 100644 --- a/Sources/SwiftDriver/Utilities/Triple.swift +++ b/Sources/SwiftDriver/Utilities/Triple.swift @@ -32,7 +32,7 @@ /// /// This is a port of https://github.com/apple/swift-llvm/blob/stable/include/llvm/ADT/Triple.h @dynamicMemberLookup -public struct Triple { +public struct Triple: Sendable { /// `Triple` proxies predicates from `Triple.OS`, returning `false` for an unknown OS. public subscript(dynamicMember predicate: KeyPath) -> Bool { os?[keyPath: predicate] ?? false @@ -60,7 +60,7 @@ public struct Triple { public let objectFormat: ObjectFormat? /// Represents a version that may be present in the target triple. - public struct Version: Equatable, Comparable, CustomStringConvertible { + public struct Version: Equatable, Comparable, CustomStringConvertible, Sendable { public static let zero = Version(0, 0, 0) public var major: Int @@ -126,7 +126,7 @@ public struct Triple { parser.components.resize(toCount: 4, paddingWith: "") parser.components[2] = "windows" if parsedEnv?.value.environment == nil { - if let objectFormat = parsedEnv?.value.objectFormat, objectFormat != .coff { + if let objectFormat = parsedEnv?.value.objectFormat { parser.components[3] = Substring(objectFormat.name) } else { parser.components[3] = "msvc" @@ -415,7 +415,7 @@ extension Triple { } } - public enum Arch: String, CaseIterable { + public enum Arch: String, CaseIterable, Decodable, Sendable { /// ARM (little endian): arm, armv.*, xscale case arm // ARM (big endian): armeb @@ -438,6 +438,8 @@ extension Triple { case bpfeb /// Hexagon: hexagon case hexagon + // M68k: Motorola 680x0 family + case m68k /// MIPS: mips, mipsallegrex, mipsr6 case mips /// MIPSEL: mipsel, mipsallegrexe, mipsr6el @@ -518,6 +520,8 @@ extension Triple { case renderscript32 // 64-bit RenderScript case renderscript64 + // Xtensa instruction set + case xtensa static func parse(_ archName: Substring) -> Triple.Arch? { switch archName { @@ -561,6 +565,8 @@ extension Triple { return .thumbeb case "avr": return .avr + case "m68k": + return .m68k case "msp430": return .msp430 case "mips", "mipseb", "mipsallegrex", "mipsisa32r6", "mipsr6": @@ -629,6 +635,8 @@ extension Triple { return .renderscript32 case "renderscript64": return .renderscript64 + case "xtensa": + return .xtensa case _ where archName.hasPrefix("arm") || archName.hasPrefix("thumb") || archName.hasPrefix("aarch64"): return parseARMArch(archName) @@ -818,7 +826,7 @@ extension Triple { case .arc, .arm, .armeb, .hexagon, .le32, .mips, .mipsel, .nvptx, .ppc, .r600, .riscv32, .sparc, .sparcel, .tce, .tcele, .thumb, .thumbeb, .x86, .xcore, .amdil, .hsail, .spir, .kalimba,.lanai, - .shave, .wasm32, .renderscript32, .aarch64_32: + .shave, .wasm32, .renderscript32, .aarch64_32, .m68k, .xtensa: return 32 case .aarch64, .aarch64e, .aarch64_be, .amdgcn, .bpfel, .bpfeb, .le64, .mips64, @@ -833,11 +841,11 @@ extension Triple { // MARK: - Parse SubArch extension Triple { - public enum SubArch: Hashable { + public enum SubArch: Hashable, Sendable { - public enum ARM { + public enum ARM: Sendable { - public enum Profile { + public enum Profile: Sendable { case a, r, m } @@ -905,13 +913,13 @@ extension Triple { } } - public enum Kalimba { + public enum Kalimba: Sendable { case v3 case v4 case v5 } - public enum MIPS { + public enum MIPS: Sendable { case r6 } @@ -1011,7 +1019,7 @@ extension Triple { // MARK: - Parse Vendor extension Triple { - public enum Vendor: String, CaseIterable, TripleComponent { + public enum Vendor: String, CaseIterable, TripleComponent, Sendable { case apple case pc case scei @@ -1028,6 +1036,7 @@ extension Triple { case mesa case suse case openEmbedded = "oe" + case swift fileprivate static func parse(_ component: Substring) -> Triple.Vendor? { switch component { @@ -1063,6 +1072,8 @@ extension Triple { return .suse case "oe": return .openEmbedded + case "swift": + return .swift default: return nil } @@ -1073,7 +1084,7 @@ extension Triple { // MARK: - Parse OS extension Triple { - public enum OS: String, CaseIterable, TripleComponent { + public enum OS: String, CaseIterable, TripleComponent, Sendable { case ananas case cloudABI = "cloudabi" case darwin @@ -1109,6 +1120,7 @@ extension Triple { case hurd case wasi case emscripten + case visionos = "xros" case noneOS // 'OS' suffix purely to avoid name clash with Optional.none var name: String { @@ -1191,6 +1203,8 @@ extension Triple { return .emscripten case _ where os.hasPrefix("none"): return .noneOS + case _ where os.hasPrefix("xros"): + return .visionos default: return nil } @@ -1247,7 +1261,7 @@ extension Triple { } } - public enum Environment: String, CaseIterable, Equatable { + public enum Environment: String, CaseIterable, Equatable, Sendable { case eabihf case eabi case elfv1 @@ -1343,7 +1357,7 @@ extension Triple { // MARK: - Parse Object Format extension Triple { - public enum ObjectFormat { + public enum ObjectFormat: Sendable { case coff case elf case macho @@ -1394,6 +1408,7 @@ extension Triple { case .kalimba: fallthrough case .le32: fallthrough case .le64: fallthrough + case .m68k: fallthrough case .mips: fallthrough case .mips64: fallthrough case .mips64el: fallthrough @@ -1417,7 +1432,8 @@ extension Triple { case .tce: fallthrough case .tcele: fallthrough case .thumbeb: fallthrough - case .xcore: + case .xcore: fallthrough + case .xtensa: return .elf case .ppc, .ppc64: @@ -1482,9 +1498,14 @@ extension Triple.OS { self == .watchos } + public var isVisionOS: Bool { + self == .visionos + } + + /// isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS). public var isDarwin: Bool { - isMacOSX || isiOS || isWatchOS + isMacOSX || isiOS || isWatchOS || isVisionOS } } @@ -1593,8 +1614,7 @@ extension Triple { if version.major < 10 { return nil } - - case .ios, .tvos, .watchos: + case .ios, .tvos, .watchos, .visionos: // Ignore the version from the triple. This is only handled because the // the clang driver combines OS X and IOS support into a common Darwin // toolchain that wants to know the OS X version number even when targeting @@ -1627,6 +1647,8 @@ extension Triple { version.major = arch == .aarch64 ? 7 : 5 } return version + case .visionos: + return Version(15, 0, 0) case .watchos: fatalError("conflicting triple info") default: @@ -1659,6 +1681,24 @@ extension Triple { fatalError("unexpected OS for Darwin triple") } } + + public var _visionOSVersion: Version { + switch os { + case .darwin, .macosx: + return Version(1, 0, 0) + case .visionos, .ios: + var version = self.osVersion + // Default to 1.0 + if version.major == 0 { + version.major = 1 + } + return version + case .watchos: + fatalError("conflicting triple info") + default: + fatalError("unexpected OS for Darwin triple") + } + } } // MARK: - Catalyst @@ -1706,13 +1746,21 @@ fileprivate extension Array { } } -// MARK: - Linker support +// MARK: - Fully static Linux support extension Triple { - /// Returns `true` if a given triple supports producing fully statically linked executables by providing `-static` - /// flag to the linker. This implies statically linking platform's libc, and of those that Swift supports currently - /// only Musl allows that reliably. - var supportsStaticExecutables: Bool { - self.environment == .musl - } + /// Returns `true` if this is the triple for Swift's fully statically + /// linked Linux target. + var isFullyStaticLinux: Bool { + self.vendor == .swift && self.environment == .musl + } + + /// Returns `true` if a given triple supports producing fully + /// statically linked executables by providing `-static` flag to + /// the linker. This implies statically linking platform's libc, + /// and of those that Swift supports currently only Musl allows + /// that reliably. + var supportsStaticExecutables: Bool { + self.isFullyStaticLinux + } } diff --git a/Sources/SwiftDriver/Utilities/VirtualPath.swift b/Sources/SwiftDriver/Utilities/VirtualPath.swift index 9dbe5d036..665147850 100644 --- a/Sources/SwiftDriver/Utilities/VirtualPath.swift +++ b/Sources/SwiftDriver/Utilities/VirtualPath.swift @@ -146,11 +146,11 @@ public enum VirtualPath: Hashable { case .absolute(let path): return .absolute(path.parentDirectory) case .relative(let path): - return .relative(RelativePath(path.dirname)) + return .relative(try! RelativePath(validating: path.dirname)) case .temporary(let path), .temporaryWithKnownContents(let path, _): - return .temporary(RelativePath(path.dirname)) + return .temporary(try! RelativePath(validating: path.dirname)) case .fileList(let path, _): - return .temporary(RelativePath(path.dirname)) + return .temporary(try! RelativePath(validating: path.dirname)) case .standardInput, .standardOutput: assertionFailure("Can't get directory of stdin/stdout") return self @@ -204,13 +204,13 @@ public enum VirtualPath: Hashable { case let .absolute(path): return .absolute(try AbsolutePath(validating: path.pathString + suffix)) case let .relative(path): - return .relative(RelativePath(path.pathString + suffix)) + return .relative(try RelativePath(validating: path.pathString + suffix)) case let .temporary(path): - return .temporary(RelativePath(path.pathString + suffix)) + return .temporary(try RelativePath(validating: path.pathString + suffix)) case let .temporaryWithKnownContents(path, contents): - return .temporaryWithKnownContents(RelativePath(path.pathString + suffix), contents) + return .temporaryWithKnownContents(try RelativePath(validating: path.pathString + suffix), contents) case let .fileList(path, content): - return .fileList(RelativePath(path.pathString + suffix), content) + return .fileList(try RelativePath(validating: path.pathString + suffix), content) case .standardInput, .standardOutput: assertionFailure("Can't append path component to standard in/out") return self @@ -331,6 +331,11 @@ extension VirtualPath { public static let standardOutput = Handle(-1) public static let standardInput = Handle(-2) +#if os(Windows) + public static let null = try! VirtualPath(path: "nul").intern() +#else + public static let null = try! VirtualPath(path: "/dev/null").intern() +#endif } /// An implementation of a concurrent path cache. @@ -431,29 +436,29 @@ extension VirtualPath { /// Clients are still allowed to instantiate `.temporary` `VirtualPath` values directly because of our inability to specify /// enum case access control, but are discouraged from doing so. extension VirtualPath { - public static func createUniqueTemporaryFile(_ path: RelativePath) -> VirtualPath { - let uniquedRelativePath = getUniqueTemporaryPath(for: path) + public static func createUniqueTemporaryFile(_ path: RelativePath) throws -> VirtualPath { + let uniquedRelativePath = try getUniqueTemporaryPath(for: path) return .temporary(uniquedRelativePath) } public static func createUniqueTemporaryFileWithKnownContents(_ path: RelativePath, _ data: Data) - -> VirtualPath { - let uniquedRelativePath = getUniqueTemporaryPath(for: path) + throws -> VirtualPath { + let uniquedRelativePath = try getUniqueTemporaryPath(for: path) return .temporaryWithKnownContents(uniquedRelativePath, data) } public static func createUniqueFilelist(_ path: RelativePath, _ fileList: FileList) - -> VirtualPath { - let uniquedRelativePath = getUniqueTemporaryPath(for: path) + throws -> VirtualPath { + let uniquedRelativePath = try getUniqueTemporaryPath(for: path) return .fileList(uniquedRelativePath, fileList) } - private static func getUniqueTemporaryPath(for path: RelativePath) -> RelativePath { + private static func getUniqueTemporaryPath(for path: RelativePath) throws -> RelativePath { let uniquedBaseName = Self.temporaryFileStore.getUniqueFilename(for: path.basenameWithoutExt) // Avoid introducing the the leading dot let dirName = path.dirname == "." ? "" : path.dirname let fileExtension = path.extension.map { ".\($0)" } ?? "" - return RelativePath(dirName + uniquedBaseName + fileExtension) + return try RelativePath(validating: dirName + uniquedBaseName + fileExtension) } /// A cache of created temporary files @@ -649,13 +654,13 @@ extension VirtualPath { case let .absolute(path): return .absolute(try AbsolutePath(validating: path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) case let .relative(path): - return .relative(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) + return .relative(try RelativePath(validating: path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) case let .temporary(path): - return .temporary(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) + return .temporary(try RelativePath(validating: path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) case let .temporaryWithKnownContents(path, contents): - return .temporaryWithKnownContents(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), contents) + return .temporaryWithKnownContents(try RelativePath(validating: path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), contents) case let .fileList(path, content): - return .fileList(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), content) + return .fileList(try RelativePath(validating: path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), content) case .standardInput, .standardOutput: return self } @@ -710,7 +715,7 @@ extension TSCBasic.FileSystem { } } - func readFileContents(_ path: VirtualPath) throws -> ByteString { + @_spi(Testing) public func readFileContents(_ path: VirtualPath) throws -> ByteString { try resolvingVirtualPath(path, apply: readFileContents) } @@ -726,7 +731,7 @@ extension TSCBasic.FileSystem { } } - func removeFileTree(_ path: VirtualPath) throws { + @_spi(Testing) public func removeFileTree(_ path: VirtualPath) throws { try resolvingVirtualPath(path) { absolutePath in try self.removeFileTree(absolutePath) } diff --git a/Sources/SwiftDriverExecution/MultiJobExecutor.swift b/Sources/SwiftDriverExecution/MultiJobExecutor.swift index d0672203e..e042ecc62 100644 --- a/Sources/SwiftDriverExecution/MultiJobExecutor.swift +++ b/Sources/SwiftDriverExecution/MultiJobExecutor.swift @@ -318,7 +318,7 @@ public final class MultiJobExecutor { // Throw the stub error the build didn't finish successfully. if !result.success { - throw Diagnostics.fatalError + throw Driver.ErrorDiagnostics.emitted } } diff --git a/Sources/SwiftOptions/Option.swift b/Sources/SwiftOptions/Option.swift index 3cb6598af..a2a0b6dc1 100644 --- a/Sources/SwiftOptions/Option.swift +++ b/Sources/SwiftOptions/Option.swift @@ -25,7 +25,7 @@ public struct OptionAttributes: OptionSet, Hashable { public static let doesNotAffectIncrementalBuild = OptionAttributes(rawValue: 0x20) public static let autolinkExtract = OptionAttributes(rawValue: 0x40) public static let moduleWrap = OptionAttributes(rawValue: 0x80) - public static let indent = OptionAttributes(rawValue: 0x100) + public static let synthesizeInterface = OptionAttributes(rawValue: 0x100) public static let argumentIsPath = OptionAttributes(rawValue: 0x200) public static let moduleInterface = OptionAttributes(rawValue: 0x400) public static let supplementaryOutput = OptionAttributes(rawValue: 0x800) @@ -56,6 +56,9 @@ public struct Option { /// An option with multiple arguments, which are collected by splitting /// the text directly following the spelling at each comma. case commaJoined + /// An option with multiple arguments, which the number of arguments is + /// specified by numArgs. + case multiArg } /// The spelling of the option, including any leading dashes. @@ -81,11 +84,15 @@ public struct Option { /// The group in which this option occurs. public let group: Group? + /// The number of arguments for MultiArg options. + public let numArgs: UInt + public init(_ spelling: String, _ kind: Kind, alias: Option? = nil, attributes: OptionAttributes = [], metaVar: String? = nil, helpText: String? = nil, - group: Group? = nil) { + group: Group? = nil, + numArgs: UInt = 0) { self.spelling = spelling self.kind = kind self.aliasFunction = alias.map { aliasOption in { aliasOption }} @@ -93,6 +100,7 @@ public struct Option { self.metaVar = metaVar self.helpText = helpText self.group = group + self.numArgs = numArgs } } diff --git a/Sources/SwiftOptions/OptionParsing.swift b/Sources/SwiftOptions/OptionParsing.swift index 3eac29780..bda78d5b9 100644 --- a/Sources/SwiftOptions/OptionParsing.swift +++ b/Sources/SwiftOptions/OptionParsing.swift @@ -153,7 +153,7 @@ extension OptionTable { throw OptionParseError.unknownOption( index: index - 1, argument: argument) } - parsedOptions.addOption(.DASHDASH, argument: .single("--")) + parsedOptions.addOption(.DASHDASH, argument: .multiple(Array())) arguments[index...].map { String($0) }.forEach { parsedOptions.addInput($0) } index = arguments.endIndex @@ -171,6 +171,21 @@ extension OptionTable { parsedOptions.addOption(option, argument: .single(arguments[index])) index += 1 + + case .multiArg: + if argument != option.spelling { + throw OptionParseError.unknownOption( + index: index - 1, argument: argument) + } + let endIdx = index + Int(option.numArgs) + if endIdx > arguments.endIndex { + throw OptionParseError.missingArgument( + index: index - 1, argument: argument) + } + parsedOptions.addOption(option, argument: .multiple(Array())) + arguments[index.." - case .separate, .remaining, .joinedOrSeparate: + case .separate, .remaining, .joinedOrSeparate, .multiArg: displayName += " " + (option.metaVar ?? "") } diff --git a/Sources/SwiftOptions/Options.swift b/Sources/SwiftOptions/Options.swift index 3227acbd3..1b9627a53 100644 --- a/Sources/SwiftOptions/Options.swift +++ b/Sources/SwiftOptions/Options.swift @@ -27,15 +27,20 @@ extension Option { public static let accessNotesPathEQ: Option = Option("-access-notes-path=", .joined, alias: Option.accessNotesPath, attributes: [.frontend, .argumentIsPath]) public static let accessNotesPath: Option = Option("-access-notes-path", .separate, attributes: [.frontend, .argumentIsPath], helpText: "Specify YAML file to override attributes on Swift declarations in this module") public static let aliasModuleNamesInModuleInterface: Option = Option("-alias-module-names-in-module-interface", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "When emitting a module interface, disambiguate modules using distinct alias names") - public static let allowUnstableCacheKeyForTesting: Option = Option("-allow-unstable-cache-key-for-testing", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Allow compilation caching with unstable inputs for testing purpose") + public static let allowAvailabilityPlatforms: Option = Option("-allow-availability-platforms", .separate, attributes: [], metaVar: "", helpText: "Restrict availability metadata to the given platforms, e.g. 'macOS,Swift'") + public static let allowNonResilientAccess: Option = Option("-allow-non-resilient-access", .flag, attributes: [.frontend], helpText: "Ensures all contents are generated besides exportable decls in the binary module, so non-resilient access can be allowed") public static let allowableClient: Option = Option("-allowable-client", .separate, attributes: [.frontend, .moduleInterface], metaVar: "", helpText: "Module names that are allowed to import this module") public static let alwaysCompileOutputFiles: Option = Option("-always-compile-output-files", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Always compile output files even it might not change the results") + public static let alwaysRebuildModuleDependencies: Option = Option("-always-rebuild-module-dependencies", .flag, attributes: [.helpHidden], helpText: "Always rebuild module dependencies") + public static let analyzeRequestEvaluator: Option = Option("-analyze-request-evaluator", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print out request evaluator cache statistics at the end of the compilation job") public static let analyzeRequirementMachine: Option = Option("-analyze-requirement-machine", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Print out requirement machine statistics at the end of the compilation job") public static let apiDiffDataDir: Option = Option("-api-diff-data-dir", .separate, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .argumentIsPath], metaVar: "", helpText: "Load platform and version specific API migration data files from . Ignored if -api-diff-data-file is specified.") public static let apiDiffDataFile: Option = Option("-api-diff-data-file", .separate, attributes: [.frontend, .noInteractive, .doesNotAffectIncrementalBuild, .argumentIsPath], metaVar: "", helpText: "API migration data is from ") + public static let enableAppExtensionLibrary: Option = Option("-application-extension-library", .flag, attributes: [.frontend, .noInteractive], helpText: "Restrict code to those available for App Extension Libraries") public static let enableAppExtension: Option = Option("-application-extension", .flag, attributes: [.frontend, .noInteractive], helpText: "Restrict code to those available for App Extensions") public static let AssertConfig: Option = Option("-assert-config", .separate, attributes: [.frontend, .moduleInterface], helpText: "Specify the assert_configuration replacement. Possible values are Debug, Release, Unchecked, DisableReplacement.") public static let AssumeSingleThreaded: Option = Option("-assume-single-threaded", .flag, attributes: [.helpHidden, .frontend], helpText: "Assume that code will be executed in a single-threaded environment") + public static let autoBridgingHeaderChaining: Option = Option("-auto-bridging-header-chaining", .flag, attributes: [.helpHidden, .frontend], helpText: "Automatically chaining all the bridging headers") public static let autolinkForceLoad: Option = Option("-autolink-force-load", .flag, attributes: [.helpHidden, .frontend, .moduleInterface], helpText: "Force ld to link against this module even if no symbols are used") public static let autolinkLibrary: Option = Option("-autolink-library", .separate, attributes: [.frontend, .noDriver], helpText: "Add dependent library") public static let avoidEmitModuleSourceInfo: Option = Option("-avoid-emit-module-source-info", .flag, attributes: [.noInteractive, .doesNotAffectIncrementalBuild], helpText: "don't emit Swift source info file") @@ -48,40 +53,48 @@ extension Option { public static let badFileDescriptorRetryCount: Option = Option("-bad-file-descriptor-retry-count", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Number of retrying opening a file if previous open returns a bad file descriptor error.") public static let baselineDir: Option = Option("-baseline-dir", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "The path to a directory containing baseline files: macos.json, iphoneos.json, appletvos.json, watchos.json, and iosmac.json") public static let baselinePath: Option = Option("-baseline-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "The path to the Json file that we should use as the baseline") - public static let batchScanInputFile: Option = Option("-batch-scan-input-file", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "Specify a JSON file containing modules to perform batch dependencies scanning") public static let BFEQ: Option = Option("-BF=", .joined, alias: Option.BF, attributes: [.noDriver]) public static let BF: Option = Option("-BF", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "add a directory to the baseline framework search path") public static let BIEQ: Option = Option("-BI=", .joined, alias: Option.BI, attributes: [.noDriver]) public static let BI: Option = Option("-BI", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "add a module for baseline input") - public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver], metaVar: "", helpText: "The path to a blocklist configuration file") + public static let blockAvailabilityPlatforms: Option = Option("-block-availability-platforms", .separate, attributes: [], metaVar: "", helpText: "Remove the given platforms from symbol graph availability metadata, e.g. 'macOS,Swift'") + public static let blockListFile: Option = Option("-blocklist-file", .separate, attributes: [.frontend, .noDriver, .argumentIsPath], metaVar: "", helpText: "The path to a blocklist configuration file") public static let breakageAllowlistPath: Option = Option("-breakage-allowlist-path", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "An allowlist of breakages to not complain about") public static let bridgingHeaderDirectoryForPrint: Option = Option("-bridging-header-directory-for-print", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Directory for bridging header to be printed in compatibility header") public static let bridgingHeaderPchKey: Option = Option("-bridging-header-pch-key", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Cache Key for bridging header pch") public static let bsdk: Option = Option("-bsdk", .joinedOrSeparate, attributes: [.noDriver, .argumentIsPath], helpText: "path to the baseline SDK to import frameworks") + public static let buildIdEQ: Option = Option("-build-id=", .joined, alias: Option.buildId, attributes: []) + public static let buildId: Option = Option("-build-id", .joinedOrSeparate, attributes: [], metaVar: "", helpText: "Specify the build ID argument passed to the linker") public static let buildModuleFromParseableInterface: Option = Option("-build-module-from-parseable-interface", .flag, alias: Option.compileModuleFromInterface, attributes: [.helpHidden, .frontend, .noDriver], group: .modes) public static let bypassBatchModeChecks: Option = Option("-bypass-batch-mode-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Bypass checks for batch-mode errors.") public static let bypassResilience: Option = Option("-bypass-resilience-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Ignore all checks for module resilience.") public static let cacheCompileJob: Option = Option("-cache-compile-job", .flag, attributes: [.frontend], helpText: "Enable compiler caching") public static let cacheDisableReplay: Option = Option("-cache-disable-replay", .flag, attributes: [.frontend, .cacheInvariant], helpText: "Skip loading the compilation result from cache") + public static let cacheReplayPrefixMap: Option = Option("-cache-replay-prefix-map", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], metaVar: "", helpText: "Remap paths when replaying outputs from cache") public static let candidateModuleFile: Option = Option("-candidate-module-file", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Specify Swift module may be ready to use for an interface") - public static let casFs: Option = Option("-cas-fs", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "", helpText: "Root CASID for CAS FileSystem") + public static let casBackendMode: Option = Option("-cas-backend-mode=", .joined, attributes: [.frontend, .noDriver], metaVar: "native|casid|verify", helpText: "CASBackendMode for output kind") + public static let casBackend: Option = Option("-cas-backend", .flag, attributes: [.frontend, .noDriver], helpText: "Enable using CASBackend for object file output") + public static let casEmitCasidFile: Option = Option("-cas-emit-casid-file", .flag, attributes: [.frontend, .noDriver], helpText: "Emit .casid file next to object file when CAS Backend is enabled") public static let casPath: Option = Option("-cas-path", .separate, attributes: [.frontend, .cacheInvariant], metaVar: "", helpText: "Path to CAS") - public static let casPluginOption: Option = Option("-cas-plugin-option", .separate, attributes: [.frontend], metaVar: "=