Skip to content

Commit ea3ec7f

Browse files
[Caching] Path remapping support
Teach swift-driver to remap path when needed. This involves: * Infer SDK/toolchain related path that needs to remapped * Send path remap related arguments to swift-frontend for depscanning * Send reverse map to swift-frontend for diagnostics replay * Constructing the compilation command for main module using remapped path
1 parent 94317a7 commit ea3ec7f

File tree

7 files changed

+255
-36
lines changed

7 files changed

+255
-36
lines changed

Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,35 @@ public struct Driver {
272272
let enableCaching: Bool
273273
let useClangIncludeTree: Bool
274274

275+
/// Scanner prefix mapping.
276+
let scannerPrefixMap: [AbsolutePath: AbsolutePath]
277+
let scannerPrefixMapSDK: AbsolutePath?
278+
let scannerPrefixMapToolchain: AbsolutePath?
279+
lazy var prefixMapping: [(AbsolutePath, AbsolutePath)] = {
280+
var mapping: [(AbsolutePath, AbsolutePath)] = scannerPrefixMap.map {
281+
return ($0.key, $0.value)
282+
}
283+
do {
284+
guard isFrontendArgSupported(.scannerPrefixMap) else {
285+
return []
286+
}
287+
if let sdkMapping = scannerPrefixMapSDK,
288+
let sdkPath = absoluteSDKPath {
289+
mapping.append((sdkPath, sdkMapping))
290+
}
291+
if let toolchainMapping = scannerPrefixMapToolchain {
292+
let toolchainPath = try toolchain.executableDir.parentDirectory // usr
293+
.parentDirectory // toolchain
294+
mapping.append((toolchainPath, toolchainMapping))
295+
}
296+
// The mapping needs to be sorted so the mapping is determinisitic.
297+
// The sorting order is reversed so /tmp/tmp is preferred over /tmp in remapping.
298+
return mapping.sorted { $0.0 > $1.0 }
299+
} catch {
300+
return mapping.sorted { $0.0 > $1.0 }
301+
}
302+
}()
303+
275304
/// Code & data for incremental compilation. Nil if not running in incremental mode.
276305
/// Set during planning because needs the jobs to look at outputs.
277306
@_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -598,6 +627,17 @@ public struct Driver {
598627
let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
599628
self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
600629
self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
630+
self.scannerPrefixMap = try Self.computeScanningPrefixMapper(&parsedOptions)
631+
if let sdkMapping = parsedOptions.getLastArgument(.scannerPrefixMapSdk)?.asSingle {
632+
self.scannerPrefixMapSDK = try AbsolutePath(validating: sdkMapping)
633+
} else {
634+
self.scannerPrefixMapSDK = nil
635+
}
636+
if let toolchainMapping = parsedOptions.getLastArgument(.scannerPrefixMapToolchain)?.asSingle {
637+
self.scannerPrefixMapToolchain = try AbsolutePath(validating: toolchainMapping)
638+
} else {
639+
self.scannerPrefixMapToolchain = nil
640+
}
601641

602642
// Compute the working directory.
603643
workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
@@ -3502,4 +3542,18 @@ extension Driver {
35023542
}
35033543
return options
35043544
}
3545+
3546+
static func computeScanningPrefixMapper(_ parsedOptions: inout ParsedOptions) throws -> [AbsolutePath: AbsolutePath] {
3547+
var mapping: [AbsolutePath: AbsolutePath] = [:]
3548+
for opt in parsedOptions.arguments(for: .scannerPrefixMap) {
3549+
let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
3550+
if pluginArg.count != 2 {
3551+
throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle)
3552+
}
3553+
let key = try AbsolutePath(validating: String(pluginArg[0]))
3554+
let value = try AbsolutePath(validating: String(pluginArg[1]))
3555+
mapping[key] = value
3556+
}
3557+
return mapping
3558+
}
35053559
}

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@ public extension Driver {
124124
try commandLine.appendLast(.clangScannerModuleCachePath, from: &parsedOptions)
125125
}
126126

127+
if isFrontendArgSupported(.scannerPrefixMap) {
128+
// construct `-scanner-prefix-mapper` for scanner.
129+
for (key, value) in prefixMapping {
130+
commandLine.appendFlag(.scannerPrefixMap)
131+
commandLine.appendFlag(key.pathString + "=" + value.pathString)
132+
}
133+
}
134+
127135
// Pass on the input files
128136
commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) })
129137
return (inputs, commandLine)

Sources/SwiftDriver/Jobs/CompileJob.swift

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,21 +112,21 @@ extension Driver {
112112
outputType: FileType?,
113113
commandLine: inout [Job.ArgTemplate])
114114
throws -> ([TypedVirtualPath], [TypedVirtualPath]) {
115-
let useInputFileList: Bool
116-
if let allSourcesFileList = allSourcesFileList {
117-
useInputFileList = true
115+
let useInputFileList = shouldUseInputFileList
116+
if shouldUseInputFileList {
117+
let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation)
118+
let remappedSourcesFileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "sources"),
119+
.list(swiftInputs.map{ return remapPath($0.file) }))
118120
commandLine.appendFlag(.filelist)
119-
commandLine.appendPath(allSourcesFileList)
120-
} else {
121-
useInputFileList = false
121+
commandLine.appendPath(remappedSourcesFileList)
122122
}
123123

124124
let usePrimaryInputFileList = primaryInputs.count > fileListThreshold
125125
if usePrimaryInputFileList {
126126
// primary file list
127127
commandLine.appendFlag(.primaryFilelist)
128128
let fileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "primaryInputs"),
129-
.list(primaryInputs.map(\.file)))
129+
.list(primaryInputs.map{ return remapPath($0.file) }))
130130
commandLine.appendPath(fileList)
131131
}
132132

@@ -166,12 +166,11 @@ extension Driver {
166166
let isPrimary = usesPrimaryFileInputs && primaryInputFiles.contains(input)
167167
if isPrimary {
168168
if !usePrimaryInputFileList {
169-
commandLine.appendFlag(.primaryFile)
170-
commandLine.appendPath(input.file)
169+
try addPathOption(option: .primaryFile, path: input.file, to:&commandLine)
171170
}
172171
} else {
173172
if !useInputFileList {
174-
commandLine.appendPath(input.file)
173+
try addPathArgument(input.file, to: &commandLine)
175174
}
176175
}
177176

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

Lines changed: 114 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ extension Driver {
7575
break
7676
}
7777

78+
let jobNeedPathRemap: Bool
7879
// If in ExplicitModuleBuild mode and the dependency graph has been computed, add module
7980
// dependencies.
8081
// May also be used for generation of the dependency graph itself in ExplicitModuleBuild mode.
@@ -83,17 +84,21 @@ extension Driver {
8384
switch kind {
8485
case .generatePCH:
8586
try addExplicitPCHBuildArguments(inputs: &inputs, commandLine: &commandLine)
87+
jobNeedPathRemap = true
8688
case .compile, .emitModule, .interpret, .verifyModuleInterface:
8789
try addExplicitModuleBuildArguments(inputs: &inputs, commandLine: &commandLine)
90+
jobNeedPathRemap = true
8891
case .backend, .mergeModule, .compileModuleFromInterface,
8992
.generatePCM, .dumpPCM, .repl, .printTargetInfo,
9093
.versionRequest, .autolinkExtract, .generateDSYM,
9194
.help, .link, .verifyDebugInfo, .scanDependencies,
9295
.emitSupportedFeatures, .moduleWrap,
9396
.generateAPIBaseline, .generateABIBaseline, .compareAPIBaseline,
9497
.compareABIBaseline:
95-
break // Do not support creating from dependency scanner output.
98+
jobNeedPathRemap = false
9699
}
100+
} else {
101+
jobNeedPathRemap = false
97102
}
98103

99104
if let variant = parsedOptions.getLastArgument(.targetVariant)?.asSingle {
@@ -145,24 +150,22 @@ extension Driver {
145150
try commandLine.appendLast(.targetCpu, from: &parsedOptions)
146151

147152
if let sdkPath = frontendTargetInfo.sdkPath?.path {
148-
commandLine.appendFlag(.sdk)
149-
commandLine.append(.path(VirtualPath.lookup(sdkPath)))
153+
try addPathOption(option: .sdk, path: VirtualPath.lookup(sdkPath), to: &commandLine, remap: jobNeedPathRemap)
150154
}
151155

152156
for args: (Option, Option) in [
153157
(.visualcToolsRoot, .visualcToolsVersion),
154158
(.windowsSdkRoot, .windowsSdkVersion)
155159
] {
156-
let (rootArg, versionArg) = args
157-
if let value = parsedOptions.getLastArgument(rootArg)?.asSingle,
158-
isFrontendArgSupported(rootArg) {
159-
commandLine.appendFlag(rootArg.spelling)
160-
commandLine.appendPath(try .init(validating: value))
160+
let (rootOpt, versionOpt) = args
161+
if let rootArg = parsedOptions.last(for: rootOpt),
162+
isFrontendArgSupported(rootOpt) {
163+
try addPathOption(rootArg, to: &commandLine, remap: jobNeedPathRemap)
161164
}
162165

163-
if let value = parsedOptions.getLastArgument(versionArg)?.asSingle,
164-
isFrontendArgSupported(versionArg) {
165-
commandLine.appendFlags(versionArg.spelling, value)
166+
if let value = parsedOptions.getLastArgument(versionOpt)?.asSingle,
167+
isFrontendArgSupported(versionOpt) {
168+
commandLine.appendFlags(versionOpt.spelling, value)
166169
}
167170
}
168171

@@ -321,12 +324,14 @@ extension Driver {
321324
commandLine.appendFlag(.Xcc)
322325
commandLine.appendFlag(.workingDirectory)
323326
commandLine.appendFlag(.Xcc)
324-
commandLine.appendPath(.absolute(workingDirectory))
327+
try addPathArgument(.absolute(workingDirectory), to: &commandLine, remap: jobNeedPathRemap)
325328
}
326329

327330
// Resource directory.
328-
commandLine.appendFlag(.resourceDir)
329-
commandLine.appendPath(VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path))
331+
try addPathOption(option: .resourceDir,
332+
path: VirtualPath.lookup(frontendTargetInfo.runtimeResourcePath.path),
333+
to: &commandLine,
334+
remap: jobNeedPathRemap)
330335

331336
if self.useStaticResourceDir {
332337
commandLine.appendFlag("-use-static-resource-dir")
@@ -366,6 +371,7 @@ extension Driver {
366371
try commandLine.appendAll(.casPluginOption, from: &parsedOptions)
367372
try commandLine.appendLast(.cacheRemarks, from: &parsedOptions)
368373
}
374+
addCacheReplayMapping(to: &commandLine)
369375
if useClangIncludeTree {
370376
commandLine.appendFlag(.clangIncludeTree)
371377
}
@@ -388,16 +394,16 @@ extension Driver {
388394
// of a lookup failure.
389395
if parsedOptions.contains(.pchOutputDir) &&
390396
!parsedOptions.contains(.driverExplicitModuleBuild) {
391-
commandLine.appendPath(VirtualPath.lookup(importedObjCHeader))
397+
try addPathArgument(VirtualPath.lookup(importedObjCHeader), to:&commandLine, remap: jobNeedPathRemap)
392398
try commandLine.appendLast(.pchOutputDir, from: &parsedOptions)
393399
if !compilerMode.isSingleCompilation {
394400
commandLine.appendFlag(.pchDisableValidation)
395401
}
396402
} else {
397-
commandLine.appendPath(VirtualPath.lookup(pch))
403+
try addPathArgument(VirtualPath.lookup(pch), to:&commandLine, remap: jobNeedPathRemap)
398404
}
399405
} else {
400-
commandLine.appendPath(VirtualPath.lookup(importedObjCHeader))
406+
try addPathArgument(VirtualPath.lookup(importedObjCHeader), to:&commandLine, remap: jobNeedPathRemap)
401407
}
402408
}
403409

@@ -437,16 +443,17 @@ extension Driver {
437443
}
438444
}
439445

440-
func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate],
441-
pchCompileJob: Job?) throws {
446+
mutating func addBridgingHeaderPCHCacheKeyArguments(commandLine: inout [Job.ArgTemplate],
447+
pchCompileJob: Job?) throws {
442448
guard let pchJob = pchCompileJob, enableCaching else { return }
443449

444450
// The pch input file (the bridging header) is added as last inputs to the job.
445451
guard let inputFile = pchJob.inputs.last else { assertionFailure("no input files from pch job"); return }
446452
assert(inputFile.type == .objcHeader, "Expect objc header input type")
453+
let mappedInput = remapPath(inputFile.file).intern()
447454
let bridgingHeaderCacheKey = try interModuleDependencyOracle.computeCacheKeyForOutput(kind: .pch,
448455
commandLine: pchJob.commandLine,
449-
input: inputFile.fileHandle)
456+
input: mappedInput)
450457
commandLine.appendFlag("-bridging-header-pch-key")
451458
commandLine.appendFlag(bridgingHeaderCacheKey)
452459
}
@@ -634,7 +641,7 @@ extension Driver {
634641
var entries = [VirtualPath.Handle: [FileType: VirtualPath.Handle]]()
635642
for input in primaryInputs {
636643
if let output = inputOutputMap[input]?.first {
637-
addEntry(&entries, input: input, output: output)
644+
try addEntry(&entries, input: input, output: output)
638645
} else {
639646
// Primary inputs are expected to appear in the output file map even
640647
// if they have no corresponding outputs.
@@ -653,7 +660,7 @@ extension Driver {
653660
}
654661

655662
for flaggedPair in flaggedInputOutputPairs {
656-
addEntry(&entries, input: flaggedPair.input, output: flaggedPair.output)
663+
try addEntry(&entries, input: flaggedPair.input, output: flaggedPair.output)
657664
}
658665
// To match the legacy driver behavior, make sure we add an entry for the
659666
// file under indexing and the primary output file path.
@@ -687,14 +694,15 @@ extension Driver {
687694
try commandLine.appendLast(.symbolGraphMinimumAccessLevel, from: &parsedOptions)
688695
}
689696

690-
func addEntry(_ entries: inout [VirtualPath.Handle: [FileType: VirtualPath.Handle]], input: TypedVirtualPath?, output: TypedVirtualPath) {
697+
mutating func addEntry(_ entries: inout [VirtualPath.Handle: [FileType: VirtualPath.Handle]], input: TypedVirtualPath?, output: TypedVirtualPath) throws {
691698
let entryInput: VirtualPath.Handle
692699
if let input = input?.fileHandle, input != OutputFileMap.singleInputKey {
693700
entryInput = input
694701
} else {
695702
entryInput = inputFiles[0].fileHandle
696703
}
697-
entries[entryInput, default: [:]][output.type] = output.fileHandle
704+
let inputEntry = enableCaching ? remapPath(VirtualPath.lookup(entryInput)).intern() : entryInput
705+
entries[inputEntry, default: [:]][output.type] = output.fileHandle
698706
}
699707

700708
/// Adds all dependencies required for an explicit module build
@@ -731,3 +739,85 @@ extension Driver {
731739
return job.moduleName == moduleOutputInfo.name
732740
}
733741
}
742+
743+
extension Driver {
744+
private func getAbsolutePathFromVirtualPath(_ path: VirtualPath) -> AbsolutePath? {
745+
guard let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory else {
746+
return nil
747+
}
748+
return path.resolvedRelativePath(base: cwd).absolutePath
749+
}
750+
751+
private mutating func remapPath(absolute path: AbsolutePath) -> AbsolutePath {
752+
guard !prefixMapping.isEmpty else {
753+
return path
754+
}
755+
for (prefix, value) in prefixMapping {
756+
if path.isDescendantOfOrEqual(to: prefix) {
757+
return value.appending(path.relative(to: prefix))
758+
}
759+
}
760+
return path
761+
}
762+
763+
public mutating func remapPath(_ path: VirtualPath) -> VirtualPath {
764+
guard !prefixMapping.isEmpty,
765+
let absPath = getAbsolutePathFromVirtualPath(path) else {
766+
return path
767+
}
768+
let mappedPath = remapPath(absolute: absPath)
769+
return try! VirtualPath(path: mappedPath.pathString)
770+
}
771+
772+
/// Helper function to add path to commandLine. Function will validate the path, and remap the path if needed.
773+
public mutating func addPathArgument(_ path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws {
774+
guard remap && enableCaching else {
775+
commandLine.appendPath(path)
776+
return
777+
}
778+
let mappedPath = remapPath(path)
779+
commandLine.appendPath(mappedPath)
780+
}
781+
782+
public mutating func addPathOption(_ option: ParsedOption, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws {
783+
let path = try VirtualPath(path: option.argument.asSingle)
784+
try addPathOption(option: option.option, path: path, to: &commandLine, remap: remap)
785+
}
786+
787+
public mutating func addPathOption(option: Option, path: VirtualPath, to commandLine: inout [Job.ArgTemplate], remap: Bool = true) throws {
788+
commandLine.appendFlag(option)
789+
let needRemap = remap && option.attributes.contains(.argumentIsPath) &&
790+
!option.attributes.contains(.cacheInvariant)
791+
try addPathArgument(path, to: &commandLine, remap: needRemap)
792+
}
793+
794+
/// Helper function to add last argument with path to command-line.
795+
public mutating func addLastArgumentWithPath(_ options: Option...,
796+
from parsedOptions: inout ParsedOptions,
797+
to commandLine: inout [Job.ArgTemplate],
798+
remap: Bool = true) throws {
799+
guard let parsedOption = parsedOptions.last(for: options) else {
800+
return
801+
}
802+
try addPathOption(parsedOption, to: &commandLine, remap: remap)
803+
}
804+
805+
/// Helper function to add all arguments with path to command-line.
806+
public mutating func addAllArgumentsWithPath(_ options: Option...,
807+
from parsedOptions: inout ParsedOptions,
808+
to commandLine: inout [Job.ArgTemplate],
809+
remap: Bool) throws {
810+
for matching in parsedOptions.arguments(for: options) {
811+
try addPathOption(matching, to: &commandLine, remap: remap)
812+
}
813+
}
814+
815+
public mutating func addCacheReplayMapping(to commandLine: inout [Job.ArgTemplate]) {
816+
if enableCaching && isFrontendArgSupported(.scannerPrefixMap) {
817+
for (key, value) in prefixMapping {
818+
commandLine.appendFlag("-cache-replay-prefix-map")
819+
commandLine.appendFlag(value.pathString + "=" + key.pathString)
820+
}
821+
}
822+
}
823+
}

0 commit comments

Comments
 (0)