File Hashing for Incremental Builds

Sources/SwiftDriver/Driver/Driver.swift

@@ -24,6 +24,7 @@ import struct TSCBasic.ByteString
 import struct TSCBasic.Diagnostic
 import struct TSCBasic.FileInfo
 import struct TSCBasic.RelativePath
+import struct TSCBasic.SHA256
 import var TSCBasic.localFileSystem
 import var TSCBasic.stderrStream
 import var TSCBasic.stdoutStream
@@ -44,6 +45,18 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible {
   }
 }
 
+public struct FileMetadata {
+    public let mTime: TimePoint
+    public let hash: String?
+    init(mTime: TimePoint, hash: String? = nil) {
+        self.mTime = mTime
+        if let hash = hash, !hash.isEmpty {
+          self.hash = hash
+        } else {
+          self.hash = nil
+        }
+    }
+}
 
 /// The Swift driver.
 public struct Driver {
@@ -209,8 +222,8 @@ public struct Driver {
   /// The set of input files
   @_spi(Testing) public let inputFiles: [TypedVirtualPath]
 
-  /// The last time each input file was modified, recorded at the start of the build.
-  @_spi(Testing) public let recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+  /// The last time each input file was modified, and the file's SHA256 hash, recorded at the start of the build.
+  @_spi(Testing) public let recordedInputMetadata: [TypedVirtualPath: FileMetadata]
 
   /// The mapping from input files to output files for each kind.
   let outputFileMap: OutputFileMap?
@@ -960,11 +973,20 @@ public struct Driver {
     // Classify and collect all of the input files.
     let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem)
     self.inputFiles = inputFiles
-    self.recordedInputModificationDates = .init(uniqueKeysWithValues:
-      Set(inputFiles).compactMap {
-        guard let modTime = try? fileSystem
-          .lastModificationTime(for: $0.file) else { return nil }
-        return ($0, modTime)
+
+    let incrementalFileHashes = parsedOptions.hasFlag(positive: .enableIncrementalFileHashing,
+                                                      negative: .disableIncrementalFileHashing,
+                                                      default: false)
+    self.recordedInputMetadata = .init(uniqueKeysWithValues:
+      Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in 
+        guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil }
+        if incrementalFileHashes {
+            guard let data = try? fileSystem.readFileContents(inputFile.file)  else { return nil }
+            let hash = SHA256().hash(data).hexadecimalRepresentation
+            return (inputFile, FileMetadata(mTime: modTime, hash: hash))
+        } else {
+            return (inputFile, FileMetadata(mTime: modTime))
+        }
     })
 
     do {
@@ -1083,7 +1105,7 @@ public struct Driver {
       outputFileMap: outputFileMap,
       incremental: self.shouldAttemptIncrementalCompilation,
       parsedOptions: parsedOptions,
-      recordedInputModificationDates: recordedInputModificationDates)
+      recordedInputMetadata: recordedInputMetadata)
 
     self.supportedFrontendFlags =
       try Self.computeSupportedCompilerArgs(of: self.toolchain,
@@ -1933,7 +1955,7 @@ extension Driver {
       }
       try executor.execute(job: inPlaceJob,
                            forceResponseFiles: forceResponseFiles,
-                           recordedInputModificationDates: recordedInputModificationDates)
+                           recordedInputMetadata: recordedInputMetadata)
     }
 
     // If requested, warn for options that weren't used by the driver after the build is finished.
@@ -1978,7 +2000,7 @@ extension Driver {
       delegate: jobExecutionDelegate,
       numParallelJobs: numParallelJobs ?? 1,
       forceResponseFiles: forceResponseFiles,
-      recordedInputModificationDates: recordedInputModificationDates)
+      recordedInputMetadata: recordedInputMetadata)
   }
 
   public func writeIncrementalBuildInformation(_ jobs: [Job]) {

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -23,13 +23,25 @@ public protocol DriverExecutor {
 
   /// Execute a single job and capture the output.
   @discardableResult
+  func execute(job: Job,
+               forceResponseFiles: Bool,
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
+
   func execute(job: Job,
                forceResponseFiles: Bool,
                recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult
 
+
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
   /// Pass in the `IncrementalCompilationState` to allow for incremental compilation.
   /// Pass in the `InterModuleDependencyGraph` to allow for module dependency tracking.
+  func execute(workload: DriverExecutorWorkload,
+               delegate: JobExecutionDelegate,
+               numParallelJobs: Int,
+               forceResponseFiles: Bool,
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+  ) throws
+
   func execute(workload: DriverExecutorWorkload,
                delegate: JobExecutionDelegate,
                numParallelJobs: Int,
@@ -38,6 +50,13 @@ public protocol DriverExecutor {
   ) throws
 
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
+  func execute(jobs: [Job],
+               delegate: JobExecutionDelegate,
+               numParallelJobs: Int,
+               forceResponseFiles: Bool,
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+  ) throws
+
   func execute(jobs: [Job],
                delegate: JobExecutionDelegate,
                numParallelJobs: Int,
@@ -53,6 +72,34 @@ public protocol DriverExecutor {
   func description(of job: Job, forceResponseFiles: Bool) throws -> String
 }
 
+extension DriverExecutor {
+    public func execute(job: Job,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
+    {
+        return try execute(job: job, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+    }
+
+
+    public func execute(workload: DriverExecutorWorkload,
+                 delegate: JobExecutionDelegate,
+                 numParallelJobs: Int,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+    ) throws {
+       try execute(workload: workload, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+    }
+
+    public func execute(jobs: [Job],
+                 delegate: JobExecutionDelegate,
+                 numParallelJobs: Int,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+    ) throws {
+        try execute(jobs: jobs, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+  }
+}
+
 public struct DriverExecutorWorkload {
   public let continueBuildingAfterErrors: Bool
   public enum Kind {
@@ -96,10 +143,10 @@ extension DriverExecutor {
   func execute<T: Decodable>(job: Job,
                              capturingJSONOutputAs outputType: T.Type,
                              forceResponseFiles: Bool,
-                             recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> T {
+                             recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> T {
     let result = try execute(job: job,
                              forceResponseFiles: forceResponseFiles,
-                             recordedInputModificationDates: recordedInputModificationDates)
+                             recordedInputMetadata: recordedInputMetadata)
 
     if (result.exitStatus != .terminated(code: EXIT_SUCCESS)) {
       let returnCode = Self.computeReturnCode(exitStatus: result.exitStatus)

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -266,7 +266,7 @@ public extension Driver {
         try self.executor.execute(job: preScanJob,
                                   capturingJSONOutputAs: InterModuleDependencyImports.self,
                                   forceResponseFiles: forceResponseFiles,
-                                  recordedInputModificationDates: recordedInputModificationDates)
+                                  recordedInputMetadata: recordedInputMetadata)
     }
     return imports
   }
@@ -343,7 +343,7 @@ public extension Driver {
         try self.executor.execute(job: scannerJob,
                                   capturingJSONOutputAs: InterModuleDependencyGraph.self,
                                   forceResponseFiles: forceResponseFiles,
-                                  recordedInputModificationDates: recordedInputModificationDates)
+                                  recordedInputMetadata: recordedInputMetadata)
     }
     return dependencyGraph
   }

Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift

@@ -46,7 +46,7 @@ extension BuildRecord {
   init(jobs: [Job],
        finishedJobResults: [BuildRecordInfo.JobResult],
        skippedInputs: Set<TypedVirtualPath>?,
-       compilationInputModificationDates: [TypedVirtualPath: TimePoint],
+       compilationInputModificationDates: [TypedVirtualPath: FileMetadata],
        actualSwiftVersion: String,
        argsHash: String,
        timeBeforeFirstJob: TimePoint,
@@ -57,10 +57,10 @@ extension BuildRecord {
         entry.job.inputsGeneratingCode.map { ($0, entry.result) }
     })
     let inputInfosArray = compilationInputModificationDates
-      .map { input, modDate -> (VirtualPath, InputInfo) in
+      .map { input, metadata -> (VirtualPath, InputInfo) in
         let status = InputInfo.Status(  wasSkipped: skippedInputs?.contains(input),
                                         jobResult: jobResultsByInput[input])
-        return (input.file, InputInfo(status: status, previousModTime: modDate))
+        return (input.file, InputInfo(status: status, previousModTime: metadata.mTime, hash: metadata.hash))
       }
 
     self.init(

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -48,7 +48,7 @@ import class Dispatch.DispatchQueue
   @_spi(Testing) public let actualSwiftVersion: String
   @_spi(Testing) public let timeBeforeFirstJob: TimePoint
   let diagnosticEngine: DiagnosticsEngine
-  let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
+  let compilationInputModificationDates: [TypedVirtualPath: FileMetadata]
   private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
 
   private var finishedJobResults = [JobResult]()
@@ -64,7 +64,7 @@ import class Dispatch.DispatchQueue
     actualSwiftVersion: String,
     timeBeforeFirstJob: TimePoint,
     diagnosticEngine: DiagnosticsEngine,
-    compilationInputModificationDates: [TypedVirtualPath: TimePoint])
+    compilationInputModificationDates: [TypedVirtualPath: FileMetadata])
   {
     self.buildRecordPath = buildRecordPath
     self.fileSystem = fileSystem
@@ -85,7 +85,7 @@ import class Dispatch.DispatchQueue
     outputFileMap: OutputFileMap?,
     incremental: Bool,
     parsedOptions: ParsedOptions,
-    recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+    recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) {
     // Cannot write a buildRecord without a path.
     guard let buildRecordPath = try? Self.computeBuildRecordPath(
@@ -99,7 +99,7 @@ import class Dispatch.DispatchQueue
     }
     let currentArgsHash = BuildRecordArguments.computeHash(parsedOptions)
     let compilationInputModificationDates =
-      recordedInputModificationDates.filter { input, _ in
+      recordedInputMetadata.filter { input, _ in
         input.type.isPartOfSwiftCompilation
       }
 

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -26,6 +26,7 @@ extension IncrementalCompilationState {
     let showJobLifecycle: Bool
     let alwaysRebuildDependents: Bool
     let interModuleDependencyGraph: InterModuleDependencyGraph?
+    let useHashes: Bool
     /// If non-null outputs information for `-driver-show-incremental` for input path
     private let reporter: Reporter?
 
@@ -45,6 +46,7 @@ extension IncrementalCompilationState {
       self.showJobLifecycle = driver.showJobLifecycle
       self.alwaysRebuildDependents = initialState.incrementalOptions.contains(
         .alwaysRebuildDependents)
+      self.useHashes = initialState.incrementalOptions.contains(.useFileHashesInModuleDependencyGraph)
       self.interModuleDependencyGraph = interModuleDependencyGraph
       self.reporter = reporter
     }
@@ -301,8 +303,9 @@ extension IncrementalCompilationState.FirstWaveComputer {
     /// The status of the input file.
     let status: InputInfo.Status
     /// If `true`, the modification time of this input matches the modification
-    /// time recorded from the prior build in the build record.
-    let datesMatch: Bool
+    /// time recorded from the prior build in the build record, or the hash of
+    /// its contents match.
+    let metadataMatch: Bool
   }
 
   // Find the inputs that have changed since last compilation, or were marked as needed a build
@@ -311,16 +314,21 @@ extension IncrementalCompilationState.FirstWaveComputer {
   ) -> [ChangedInput] {
     jobsInPhases.compileJobs.compactMap { job in
       let input = job.primaryInputs[0]
-      let modDate = buildRecordInfo.compilationInputModificationDates[input] ?? .distantFuture
+      let metadata = buildRecordInfo.compilationInputModificationDates[input] ?? FileMetadata(mTime: .distantFuture)
       let inputInfo = outOfDateBuildRecord.inputInfos[input.file]
       let previousCompilationStatus = inputInfo?.status ?? .newlyAdded
       let previousModTime = inputInfo?.previousModTime
+      let previousHash = inputInfo?.hash 
+
+      assert(metadata.hash != nil || !useHashes)
 
       switch previousCompilationStatus {
-      case .upToDate where modDate == previousModTime:
+      case .upToDate where metadata.mTime == previousModTime:
         reporter?.report("May skip current input:", input)
         return nil
-
+      case .upToDate where useHashes && (metadata.hash == previousHash):
+        reporter?.report("May skip current input (identical hash):", input)
+        return nil
       case .upToDate:
         reporter?.report("Scheduling changed input", input)
       case .newlyAdded:
@@ -330,9 +338,10 @@ extension IncrementalCompilationState.FirstWaveComputer {
       case .needsNonCascadingBuild:
         reporter?.report("Scheduling noncascading build", input)
       }
+      let metadataMatch = metadata.mTime == previousModTime || (useHashes && metadata.hash == previousHash)
       return ChangedInput(typedFile: input,
                           status: previousCompilationStatus,
-                          datesMatch: modDate == previousModTime)
+                          metadataMatch: metadataMatch )
     }
   }
 
@@ -381,7 +390,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
   ) -> [TypedVirtualPath] {
     changedInputs.compactMap { changedInput in
       let inputIsUpToDate =
-        changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
+        changedInput.metadataMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
       let basename = changedInput.typedFile.file.basename
 
       // If we're asked to always rebuild dependents, all we need to do is

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

@@ -373,6 +373,10 @@ extension IncrementalCompilationState {
     /// Enables additional handling of explicit module build artifacts:
     /// Additional reading and writing of the inter-module dependency graph.
     public static let explicitModuleBuild                    = Options(rawValue: 1 << 6)
+
+    /// Enables use of file hashes as a fallback in the case that a timestamp 
+    /// change might invalidate a node
+    public static let useFileHashesInModuleDependencyGraph  = Options(rawValue: 1 << 7)
   }
 }
 

Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift

@@ -89,6 +89,12 @@ extension IncrementalCompilationState {
     if driver.parsedOptions.contains(.driverExplicitModuleBuild) {
       options.formUnion(.explicitModuleBuild)
     }
+    if driver.parsedOptions.hasFlag(positive: .enableIncrementalFileHashing,
+                                    negative: .disableIncrementalFileHashing,
+                                    default: false) {
+      options.formUnion(.useFileHashesInModuleDependencyGraph)
+    }
+
     return options
   }
 }
@@ -128,6 +134,9 @@ extension IncrementalCompilationState {
     @_spi(Testing) public var emitDependencyDotFileAfterEveryImport: Bool {
       options.contains(.emitDependencyDotFileAfterEveryImport)
     }
+    @_spi(Testing) public var useFileHashesInModuleDependencyGraph: Bool {
+      options.contains(.useFileHashesInModuleDependencyGraph)
+    }
 
     @_spi(Testing) public init(
       _ options: Options,

Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import struct TSCBasic.ProcessResult
+import struct TSCBasic.SHA256
 
 /// Contains information about the current status of an input to the incremental
 /// build.
@@ -25,9 +26,12 @@ import struct TSCBasic.ProcessResult
   /// The last known modification time of this input.
   /*@_spi(Testing)*/ public let previousModTime: TimePoint
 
-  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint) {
+  /*@_spi(Testing)*/ public let hash: String?
+
+  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String?) {
     self.status = status
     self.previousModTime = previousModTime
+    self.hash = hash
   }
 }