Skip to content

Commit f7e93e5

Browse files
cachemeifyoucancachemeifyoucan
committed
[CAS] Don't create the same CAS multiple times from the same Oracle
If the oracle has been used to create a CAS for the path, reuse the CAS if it is asked to create the same CAS again. This not only save the work of creating the CAS, but also avoid a potential problem that the CAS closing synchronization doesn't work across threads in the same process due to its uses of file lock.
1 parent 8e47ffe commit f7e93e5

File tree

6 files changed

+73
-22
lines changed

6 files changed

+73
-22
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ public struct Driver {
275275
let useClangIncludeTree: Bool
276276

277277
/// CAS instance used for compilation.
278-
var cas: SwiftScanCAS? = nil
278+
@_spi(Testing) public var cas: SwiftScanCAS? = nil
279279

280280
/// Is swift caching enabled.
281281
lazy var isCachingEnabled: Bool = {

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,11 +165,27 @@ public class InterModuleDependencyOracle {
165165
return diags.isEmpty ? nil : diags
166166
}
167167

168-
public func createCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
168+
public func getOrCreateCAS(pluginPath: AbsolutePath?, onDiskPath: AbsolutePath?, pluginOptions: [(String, String)]) throws -> SwiftScanCAS {
169169
guard let swiftScan = swiftScanLibInstance else {
170170
fatalError("Attempting to reset scanner cache with no scanner instance.")
171171
}
172-
return try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions)
172+
// Use synchronized queue to avoid creating multiple OnDisk CAS at the same location as that will leave to synchronization issues.
173+
return try queue.sync {
174+
if let path = onDiskPath {
175+
if let casConfig = createdCASMap[path] {
176+
assert(casConfig.onDiskPath == onDiskPath, "onDiskPath not equal")
177+
guard casConfig.pluginPath == pluginPath, casConfig.pluginOptions.elementsEqual(casConfig.pluginOptions, by: ==) else {
178+
throw DependencyScanningError.casError("Cannot create two different CAS at the same OnDiskPath")
179+
}
180+
return casConfig.cas
181+
}
182+
let cas = try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: path.pathString, pluginOptions: pluginOptions)
183+
createdCASMap[path] = CASConfig(onDiskPath: onDiskPath, pluginPath: pluginPath, pluginOptions: pluginOptions, cas: cas)
184+
return cas
185+
}
186+
// If onDiskPath is nil, just create a new CAS.
187+
return try swiftScan.createCAS(pluginPath: pluginPath?.pathString, onDiskPath: onDiskPath?.pathString, pluginOptions: pluginOptions)
188+
}
173189
}
174190

175191
private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
@@ -181,5 +197,14 @@ public class InterModuleDependencyOracle {
181197
private var swiftScanLibInstance: SwiftScan? = nil
182198

183199
internal let scannerRequiresPlaceholderModules: Bool
200+
201+
internal struct CASConfig {
202+
let onDiskPath: AbsolutePath?
203+
let pluginPath: AbsolutePath?
204+
let pluginOptions: [(String, String)]
205+
let cas: SwiftScanCAS
206+
}
207+
/// Storing the CAS created via on-disk-path.
208+
internal var createdCASMap: [AbsolutePath: CASConfig] = [:]
184209
}
185210

‎Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,9 @@ public extension Driver {
173173
}
174174
}
175175
if !fallbackToFrontend && isCachingEnabled {
176-
self.cas = try interModuleDependencyOracle.createCAS(pluginPath: try getCASPluginPath(),
177-
onDiskPath: try getOnDiskCASPath(),
178-
pluginOptions: try getCASPluginOptions())
176+
self.cas = try interModuleDependencyOracle.getOrCreateCAS(pluginPath: try getCASPluginPath(),
177+
onDiskPath: try getOnDiskCASPath(),
178+
pluginOptions: try getCASPluginOptions())
179179
}
180180
return fallbackToFrontend
181181
}

‎Sources/SwiftDriver/SwiftScan/SwiftScanCAS.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,12 @@ public final class SwiftScanCAS {
244244
}
245245
}
246246

247+
extension SwiftScanCAS: Equatable {
248+
static public func == (lhs: SwiftScanCAS, rhs: SwiftScanCAS) -> Bool {
249+
return lhs.cas == rhs.cas
250+
}
251+
}
252+
247253
extension swiftscan_cached_compilation_t {
248254
func convert(_ lib: SwiftScan) -> CachedCompilation {
249255
return CachedCompilation(self, lib: lib)

‎Tests/SwiftDriverTests/CachingBuildTests.swift‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,8 @@ private func checkCachingBuildJobDependencies(job: Job,
205205

206206

207207
final class CachingBuildTests: XCTestCase {
208+
let dependencyOracle = InterModuleDependencyOracle()
209+
208210
override func setUpWithError() throws {
209211
try super.setUpWithError()
210212

@@ -499,12 +501,12 @@ final class CachingBuildTests: XCTestCase {
499501
"-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
500502
"-working-directory", path.nativePathString(escaped: true),
501503
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
502-
env: ProcessEnv.vars)
504+
env: ProcessEnv.vars,
505+
interModuleDependencyOracle: dependencyOracle)
503506
let jobs = try driver.planBuild()
504507
try driver.run(jobs: jobs)
505508
XCTAssertFalse(driver.diagnosticEngine.hasErrors)
506509

507-
let dependencyOracle = InterModuleDependencyOracle()
508510
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
509511
guard try dependencyOracle
510512
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -513,7 +515,12 @@ final class CachingBuildTests: XCTestCase {
513515
return
514516
}
515517

516-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
518+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
519+
if let driverCAS = driver.cas {
520+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
521+
} else {
522+
XCTFail("Cached compilation doesn't have a CAS")
523+
}
517524
try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
518525
}
519526
}
@@ -556,9 +563,9 @@ final class CachingBuildTests: XCTestCase {
556563
"-pch-output-dir", PCHPath.nativePathString(escaped: true),
557564
FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)]
558565
+ sdkArgumentsForTesting,
559-
env: ProcessEnv.vars)
566+
env: ProcessEnv.vars,
567+
interModuleDependencyOracle: dependencyOracle)
560568

561-
let dependencyOracle = InterModuleDependencyOracle()
562569
let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib())
563570
guard try dependencyOracle
564571
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -574,7 +581,12 @@ final class CachingBuildTests: XCTestCase {
574581
try fooBuildDriver.run(jobs: fooJobs)
575582
XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors)
576583

577-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
584+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
585+
if let driverCAS = fooBuildDriver.cas {
586+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
587+
} else {
588+
XCTFail("Cached compilation doesn't have a CAS")
589+
}
578590
try checkCASForResults(jobs: fooJobs, cas: cas, fs: fooBuildDriver.fileSystem)
579591

580592
var driver = try Driver(args: ["swiftc",
@@ -585,7 +597,8 @@ final class CachingBuildTests: XCTestCase {
585597
"-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true),
586598
"-working-directory", path.nativePathString(escaped: true),
587599
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
588-
env: ProcessEnv.vars)
600+
env: ProcessEnv.vars,
601+
interModuleDependencyOracle: dependencyOracle)
589602
// This is currently not supported.
590603
XCTAssertThrowsError(try driver.planBuild()) {
591604
XCTAssertEqual($0 as? Driver.Error, .unsupportedConfigurationForCaching("module Foo has prebuilt header dependency"))
@@ -623,8 +636,8 @@ final class CachingBuildTests: XCTestCase {
623636
"-Xcc", "-ivfsoverlay", "-Xcc", vfsoverlay.nativePathString(escaped: true),
624637
"-disable-clang-target",
625638
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
626-
env: ProcessEnv.vars)
627-
let dependencyOracle = InterModuleDependencyOracle()
639+
env: ProcessEnv.vars,
640+
interModuleDependencyOracle: dependencyOracle)
628641
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
629642
guard try dependencyOracle
630643
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -760,11 +773,11 @@ final class CachingBuildTests: XCTestCase {
760773
"-scanner-prefix-map", testInputsPath.description + "=/^src",
761774
"-scanner-prefix-map", path.description + "=/^tmp",
762775
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
763-
env: ProcessEnv.vars)
776+
env: ProcessEnv.vars,
777+
interModuleDependencyOracle: dependencyOracle)
764778
guard driver.isFrontendArgSupported(.scannerPrefixMap) else {
765779
throw XCTSkip("frontend doesn't support prefix map")
766780
}
767-
let dependencyOracle = InterModuleDependencyOracle()
768781
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
769782
guard try dependencyOracle
770783
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -830,12 +843,12 @@ final class CachingBuildTests: XCTestCase {
830843
"-output-file-map", ofm.nativePathString(escaped: true),
831844
"-working-directory", path.nativePathString(escaped: true),
832845
main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
833-
env: ProcessEnv.vars)
846+
env: ProcessEnv.vars,
847+
interModuleDependencyOracle: dependencyOracle)
834848
let jobs = try driver.planBuild()
835849
try driver.run(jobs: jobs)
836850
XCTAssertFalse(driver.diagnosticEngine.hasErrors)
837851

838-
let dependencyOracle = InterModuleDependencyOracle()
839852
let scanLibPath = try XCTUnwrap(driver.toolchain.lookupSwiftScanLib())
840853
guard try dependencyOracle
841854
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -844,7 +857,12 @@ final class CachingBuildTests: XCTestCase {
844857
return
845858
}
846859

847-
let cas = try dependencyOracle.createCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
860+
let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: [])
861+
if let driverCAS = driver.cas {
862+
XCTAssertEqual(cas, driverCAS, "CAS should only be created once")
863+
} else {
864+
XCTFail("Cached compilation doesn't have a CAS")
865+
}
848866
try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem)
849867

850868
// try replan the job and make sure some key command-line options are generated.

‎Tests/TestUtilities/DriverExtensions.swift‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ extension Driver {
2323
env: [String: String] = ProcessEnv.vars,
2424
diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
2525
fileSystem: FileSystem = localFileSystem,
26-
integratedDriver: Bool = true
26+
integratedDriver: Bool = true,
27+
interModuleDependencyOracle: InterModuleDependencyOracle? = nil
2728
) throws {
2829
let executor = try SwiftDriverExecutor(diagnosticsEngine: diagnosticsEngine,
2930
processSet: ProcessSet(),
@@ -34,7 +35,8 @@ extension Driver {
3435
diagnosticsOutput: .engine(diagnosticsEngine),
3536
fileSystem: fileSystem,
3637
executor: executor,
37-
integratedDriver: integratedDriver)
38+
integratedDriver: integratedDriver,
39+
interModuleDependencyOracle: interModuleDependencyOracle)
3840
}
3941

4042
/// For tests that need to set the sdk path.

0 commit comments

Comments
 (0)