Skip to content

Commit 893f014

Browse files
Merge pull request #1393 from cachemeifyoucan/eng/PR-swift-caching-support
Support Swift CompileJob Caching
2 parents d01fb4c + f3ecb9e commit 893f014

23 files changed

+1329
-83
lines changed

Sources/CSwiftScan/include/swiftscan_header.h

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <stdint.h>
1919

2020
#define SWIFTSCAN_VERSION_MAJOR 0
21-
#define SWIFTSCAN_VERSION_MINOR 1
21+
#define SWIFTSCAN_VERSION_MINOR 4
2222

2323
//=== Public Scanner Data Types -------------------------------------------===//
2424

@@ -77,6 +77,18 @@ typedef struct {
7777
typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t;
7878
typedef void *swiftscan_scanner_t;
7979

80+
//=== CAS/Caching Specification -------------------------------------------===//
81+
typedef struct swiftscan_cas_s *swiftscan_cas_t;
82+
83+
typedef enum {
84+
SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
85+
SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
86+
SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
87+
SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3,
88+
SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
89+
SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
90+
} swiftscan_output_kind_t;
91+
8092
//=== libSwiftScan Functions ------------------------------------------------===//
8193

8294
typedef struct {
@@ -117,13 +129,17 @@ typedef struct {
117129
swiftscan_string_set_t *
118130
(*swiftscan_swift_textual_detail_get_command_line)(swiftscan_module_details_t);
119131
swiftscan_string_set_t *
132+
(*swiftscan_swift_textual_detail_get_bridging_pch_command_line)(swiftscan_module_details_t);
133+
swiftscan_string_set_t *
120134
(*swiftscan_swift_textual_detail_get_extra_pcm_args)(swiftscan_module_details_t);
121135
swiftscan_string_ref_t
122136
(*swiftscan_swift_textual_detail_get_context_hash)(swiftscan_module_details_t);
123137
bool
124138
(*swiftscan_swift_textual_detail_get_is_framework)(swiftscan_module_details_t);
125139
swiftscan_string_set_t *
126140
(*swiftscan_swift_textual_detail_get_swift_overlay_dependencies)(swiftscan_module_details_t);
141+
swiftscan_string_ref_t
142+
(*swiftscan_swift_textual_detail_get_module_cache_key)(swiftscan_module_details_t);
127143

128144
//=== Swift Binary Module Details query APIs ------------------------------===//
129145
swiftscan_string_ref_t
@@ -136,6 +152,8 @@ typedef struct {
136152
(*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t);
137153
bool
138154
(*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t);
155+
swiftscan_string_ref_t
156+
(*swiftscan_swift_binary_detail_get_module_cache_key)(swiftscan_module_details_t);
139157

140158
//=== Swift Placeholder Module Details query APIs -------------------------===//
141159
swiftscan_string_ref_t
@@ -154,6 +172,8 @@ typedef struct {
154172
(*swiftscan_clang_detail_get_command_line)(swiftscan_module_details_t);
155173
swiftscan_string_set_t *
156174
(*swiftscan_clang_detail_get_captured_pcm_args)(swiftscan_module_details_t);
175+
swiftscan_string_ref_t
176+
(*swiftscan_clang_detail_get_module_cache_key)(swiftscan_module_details_t);
157177

158178
//=== Batch Scan Input Functions ------------------------------------------===//
159179
swiftscan_batch_scan_input_t *
@@ -253,6 +273,17 @@ typedef struct {
253273
bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
254274
void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
255275

276+
//=== Scanner CAS Operations ----------------------------------------------===//
277+
swiftscan_cas_t (*swiftscan_cas_create)(const char *path);
278+
void (*swiftscan_cas_dispose)(swiftscan_cas_t cas);
279+
swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas,
280+
uint8_t *data, unsigned size);
281+
swiftscan_string_ref_t (*swiftscan_compute_cache_key)(swiftscan_cas_t cas,
282+
int argc,
283+
const char *argv,
284+
const char *input,
285+
swiftscan_output_kind_t);
286+
256287
} swiftscan_functions_t;
257288

258289
#endif // SWIFT_C_DEPENDENCY_SCAN_H

Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ public struct Driver {
7474
case missingContextHashOnSwiftDependency(String)
7575
case dependencyScanningFailure(Int, String)
7676
case missingExternalDependency(String)
77+
// Compiler Caching Failures
78+
case unsupportedConfigurationForCaching(String)
7779

7880
public var description: String {
7981
switch self {
@@ -135,6 +137,8 @@ public struct Driver {
135137
return "unable to load output file map '\(path)': \(error)"
136138
case .missingExternalDependency(let moduleName):
137139
return "Missing External dependency info for module: \(moduleName)"
140+
case .unsupportedConfigurationForCaching(let reason):
141+
return "unsupported configuration for -cache-compile-job: \(reason)"
138142
case .baselineGenerationRequiresTopLevelModule(let arg):
139143
return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'"
140144
case .optionRequiresAnother(let first, let second):
@@ -263,6 +267,11 @@ public struct Driver {
263267
/// Whether to consider incremental compilation.
264268
let shouldAttemptIncrementalCompilation: Bool
265269

270+
/// CAS/Caching related options.
271+
let enableCaching: Bool
272+
let useClangIncludeTree: Bool
273+
let casPath: String
274+
266275
/// Code & data for incremental compilation. Nil if not running in incremental mode.
267276
/// Set during planning because needs the jobs to look at outputs.
268277
@_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -571,6 +580,17 @@ public struct Driver {
571580
diagnosticEngine: diagnosticsEngine,
572581
compilerMode: compilerMode)
573582

583+
let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
584+
self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
585+
self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
586+
if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle {
587+
self.casPath = casPathOpt.description
588+
} else if let cacheEnv = env["CCHROOT"] {
589+
self.casPath = cacheEnv
590+
} else {
591+
self.casPath = ""
592+
}
593+
574594
// Compute the working directory.
575595
workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
576596
let cwd = fileSystem.currentWorkingDirectory

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

Lines changed: 73 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
4545
/// Whether we are using the integrated driver via libSwiftDriver shared lib
4646
private let integratedDriver: Bool
4747
private let mainModuleName: String?
48+
private let enableCAS: Bool
49+
private let swiftScanOracle: InterModuleDependencyOracle
4850

4951
/// Clang PCM names contain a hash of the command-line arguments that were used to build them.
5052
/// We avoid re-running the hash computation with the use of this cache
@@ -55,14 +57,23 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
5557

5658
public init(dependencyGraph: InterModuleDependencyGraph,
5759
toolchain: Toolchain,
60+
dependencyOracle: InterModuleDependencyOracle,
5861
integratedDriver: Bool = true,
59-
supportsExplicitInterfaceBuild: Bool = false) throws {
62+
supportsExplicitInterfaceBuild: Bool = false,
63+
enableCAS: Bool = false) throws {
6064
self.dependencyGraph = dependencyGraph
6165
self.toolchain = toolchain
66+
self.swiftScanOracle = dependencyOracle
6267
self.integratedDriver = integratedDriver
6368
self.mainModuleName = dependencyGraph.mainModuleName
6469
self.reachabilityMap = try dependencyGraph.computeTransitiveClosure()
6570
self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild
71+
self.enableCAS = enableCAS
72+
}
73+
74+
/// Supports resolving bridging header pch command from swiftScan.
75+
public func supportsBridgingHeaderPCHCommand() throws -> Bool {
76+
return try swiftScanOracle.supportsBridgingHeaderPCHCommand()
6677
}
6778

6879
/// Generate build jobs for all dependencies of the main module.
@@ -235,7 +246,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
235246
inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path,
236247
type: .swiftModule))
237248

238-
for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] {
249+
let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? []
250+
if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty {
251+
throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency")
252+
}
253+
254+
for headerDep in prebuiltHeaderDependencyPaths {
239255
commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"])
240256
commandLine.appendPath(VirtualPath.lookup(headerDep.path))
241257
inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch))
@@ -256,11 +272,21 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
256272

257273
// Swift Main Module dependencies are passed encoded in a JSON file as described by
258274
// SwiftModuleArtifactInfo
259-
if moduleId.moduleName == mainModuleName {
275+
guard moduleId == .swift(dependencyGraph.mainModuleName) else { return }
276+
let dependencyFileContent =
277+
try serializeModuleDependencies(for: moduleId,
278+
swiftDependencyArtifacts: swiftDependencyArtifacts,
279+
clangDependencyArtifacts: clangDependencyArtifacts)
280+
if enableCAS {
281+
// When using a CAS, write JSON into CAS and pass the ID on command-line.
282+
let casID = try swiftScanOracle.store(data: dependencyFileContent)
283+
commandLine.appendFlag("-explicit-swift-module-map-file")
284+
commandLine.appendFlag(casID)
285+
} else {
286+
// Write JSON to a file and add the JSON artifacts to command-line and inputs.
260287
let dependencyFile =
261-
try serializeModuleDependencies(for: moduleId,
262-
swiftDependencyArtifacts: swiftDependencyArtifacts,
263-
clangDependencyArtifacts: clangDependencyArtifacts)
288+
VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"),
289+
dependencyFileContent)
264290
commandLine.appendFlag("-explicit-swift-module-map-file")
265291
commandLine.appendPath(dependencyFile)
266292
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
@@ -280,13 +306,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
280306
let isFramework: Bool
281307
swiftModulePath = .init(file: dependencyInfo.modulePath.path,
282308
type: .swiftModule)
283-
isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework ?? false
309+
let swiftModuleDetails = try dependencyGraph.swiftModuleDetails(of: dependencyId)
310+
isFramework = swiftModuleDetails.isFramework ?? false
284311
// Accumulate the required information about this dependency
285312
// TODO: add .swiftdoc and .swiftsourceinfo for this module.
286313
swiftDependencyArtifacts.append(
287314
SwiftModuleArtifactInfo(name: dependencyId.moduleName,
288315
modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
289-
isFramework: isFramework))
316+
isFramework: isFramework,
317+
moduleCacheKey: swiftModuleDetails.moduleCacheKey))
290318
case .clang:
291319
let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId)
292320
let dependencyClangModuleDetails =
@@ -295,7 +323,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
295323
clangDependencyArtifacts.append(
296324
ClangModuleArtifactInfo(name: dependencyId.moduleName,
297325
modulePath: TextualVirtualPath(path: dependencyInfo.modulePath.path),
298-
moduleMapPath: dependencyClangModuleDetails.moduleMapPath))
326+
moduleMapPath: dependencyClangModuleDetails.moduleMapPath,
327+
moduleCacheKey: dependencyClangModuleDetails.moduleCacheKey))
299328
case .swiftPrebuiltExternal:
300329
let prebuiltModuleDetails = try dependencyGraph.swiftPrebuiltDetails(of: dependencyId)
301330
let compiledModulePath = prebuiltModuleDetails.compiledModulePath
@@ -308,7 +337,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
308337
SwiftModuleArtifactInfo(name: dependencyId.moduleName,
309338
modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle),
310339
headerDependencies: prebuiltModuleDetails.headerDependencyPaths,
311-
isFramework: isFramework))
340+
isFramework: isFramework,
341+
moduleCacheKey: prebuiltModuleDetails.moduleCacheKey))
312342
case .swiftPlaceholder:
313343
fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)")
314344
}
@@ -354,6 +384,11 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
354384
public mutating func resolveMainModuleDependencies(inputs: inout [TypedVirtualPath],
355385
commandLine: inout [Job.ArgTemplate]) throws {
356386
let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
387+
388+
let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
389+
if let additionalArgs = mainModuleDetails.commandLine {
390+
additionalArgs.forEach { commandLine.appendFlag($0) }
391+
}
357392
commandLine.appendFlags("-disable-implicit-swift-modules",
358393
"-Xcc", "-fno-implicit-modules",
359394
"-Xcc", "-fno-implicit-module-maps")
@@ -367,11 +402,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
367402
public mutating func resolveBridgingHeaderDependencies(inputs: inout [TypedVirtualPath],
368403
commandLine: inout [Job.ArgTemplate]) throws {
369404
let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
370-
// Prohibit the frontend from implicitly building textual modules into binary modules.
371-
commandLine.appendFlags("-disable-implicit-swift-modules",
372-
"-Xcc", "-fno-implicit-modules",
373-
"-Xcc", "-fno-implicit-module-maps")
374-
375405
var swiftDependencyArtifacts: [SwiftModuleArtifactInfo] = []
376406
var clangDependencyArtifacts: [ClangModuleArtifactInfo] = []
377407
let mainModuleDetails = try dependencyGraph.swiftModuleDetails(of: mainModuleId)
@@ -409,28 +439,46 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
409439
inputs.append(clangModuleMapPath)
410440
}
411441

442+
// Return if depscanner provided build commands.
443+
if let scannerPCHArgs = mainModuleDetails.bridgingPchCommandLine {
444+
scannerPCHArgs.forEach { commandLine.appendFlag($0) }
445+
return
446+
}
447+
448+
assert(!enableCAS, "Caching build should always return command-line from scanner")
449+
// Prohibit the frontend from implicitly building textual modules into binary modules.
450+
commandLine.appendFlags("-disable-implicit-swift-modules",
451+
"-Xcc", "-fno-implicit-modules",
452+
"-Xcc", "-fno-implicit-module-maps")
453+
454+
let dependencyFileContent =
455+
try serializeModuleDependencies(for: mainModuleId,
456+
swiftDependencyArtifacts: swiftDependencyArtifacts,
457+
clangDependencyArtifacts: clangDependencyArtifacts)
458+
412459
let dependencyFile =
413-
try serializeModuleDependencies(for: mainModuleId,
414-
swiftDependencyArtifacts: swiftDependencyArtifacts,
415-
clangDependencyArtifacts: clangDependencyArtifacts)
460+
VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(mainModuleId.moduleName)-dependencies.json"),
461+
dependencyFileContent)
416462
commandLine.appendFlag("-explicit-swift-module-map-file")
417463
commandLine.appendPath(dependencyFile)
418464
inputs.append(TypedVirtualPath(file: dependencyFile.intern(),
419465
type: .jsonSwiftArtifacts))
420466
}
421467

422-
/// Store the output file artifacts for a given module in a JSON file, return the file's path.
468+
/// Serialize the output file artifacts for a given module in JSON format.
423469
private func serializeModuleDependencies(for moduleId: ModuleDependencyId,
424470
swiftDependencyArtifacts: [SwiftModuleArtifactInfo],
425471
clangDependencyArtifacts: [ClangModuleArtifactInfo]
426-
) throws -> VirtualPath {
472+
) throws -> Data {
473+
// The module dependency map in CAS needs to be stable.
474+
// Sort the dependencies by name.
427475
let allDependencyArtifacts: [ModuleDependencyArtifactInfo] =
428-
swiftDependencyArtifacts.map {ModuleDependencyArtifactInfo.swift($0)} +
429-
clangDependencyArtifacts.map {ModuleDependencyArtifactInfo.clang($0)}
476+
swiftDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.swift($0)} +
477+
clangDependencyArtifacts.sorted().map {ModuleDependencyArtifactInfo.clang($0)}
430478
let encoder = JSONEncoder()
431-
encoder.outputFormatting = [.prettyPrinted]
432-
let contents = try encoder.encode(allDependencyArtifacts)
433-
return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents)
479+
// Use sorted key to ensure the order of the keys is stable.
480+
encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
481+
return try encoder.encode(allDependencyArtifacts)
434482
}
435483

436484
private func getPCMHashParts(pcmArgs: [String], contextHash: String) -> [String] {

0 commit comments

Comments
 (0)