Touch all output files (#120)

Author: dfederm

README.md

@@ -74,6 +74,7 @@ These settings are common across all plugins, although different implementations
 | `$(MSBuildCacheGetResultsForUnqueriedDependencies)` | `bool` | false | Whether to try and query the cache for dependencies if they have not previously been requested. This option can help in cases where the build isn't done in graph order, or if some projects are skipped. |
 | `$(MSBuildCacheTargetsToIgnore)` | `string[]` | `GetTargetFrameworks;GetNativeManifest;GetCopyToOutputDirectoryItems;GetTargetFrameworksWithPlatformForSingleTargetFramework` | The list of targets to ignore when determining if a build request matches a cache entry. This is intended for "information gathering" targets which do not have side-effect. eg. a build with `/t:Build` and `/t:Build;GetTargetFrameworks` should be considered to have equivalent results. Note: This only works "one-way" in that the build request is allowed to have missing targets, while the cache entry is not. This is to avoid a situation where a build request recieves a cache hit with missing target results, where a cache hit with extra target results is acceptable. |
 | `$(MSBuildCacheSkipUnchangedOutputFiles)` | `bool` | false | Whether to avoid writing output files on cache hit if the file is unchanged, which can improve performance for incremental builds. A file is considered unchanged if it exists, the previously placed file and file to be placed have the same hash, and the the previously placed file and current file on disk have the same timestamp and file size. |
+| `$(MSBuildCacheTouchOutputFiles)` | `bool` | false | Whether to update the last write time for output files on cache hit. All files for a given cache entry will have the same timestamp. Note that outputs which skip materialization via `MSBuildCacheSkipUnchangedOutputFiles` are still touched. |
 | `$(MSBuildCacheIgnoreDotNetSdkPatchVersion)` | `bool` | false | Whether to ignore the patch version when doing cache lookups. This trades off some correctness for the sake of getting cache hits when the SDK version isn't exactly the same. The default behavior is to consider the exact SDK version, eg. "8.0.404". With this setting set to true, it will instead use something like "8.0.4XX". Note that the major version, minor version, and feature bands are still considered. |
 
 When configuring settings which are list types, you should always append to the existing value to avoid overriding the defaults:

src/AzureBlobStorage/MSBuildCacheAzureBlobStoragePlugin.cs

@@ -112,7 +112,8 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
             Settings.MaxConcurrentCacheContentOperations,
             Settings.AsyncCachePublishing,
             Settings.AsyncCacheMaterialization,
-            Settings.SkipUnchangedOutputFiles);
+            Settings.SkipUnchangedOutputFiles,
+            Settings.TouchOutputFiles);
     }
 
     private async Task<ICache> CreateRemoteCacheAsync(

src/AzurePipelines/MSBuildCacheAzurePipelinesPlugin.cs

@@ -62,7 +62,8 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
             Settings.RemoteCacheIsReadOnly,
             Settings.AsyncCachePublishing,
             Settings.AsyncCacheMaterialization,
-            Settings.SkipUnchangedOutputFiles);
+            Settings.SkipUnchangedOutputFiles,
+            Settings.TouchOutputFiles);
     }
 
     private static async Task<ICacheSession> StartCacheSessionAsync(Context context, LocalCache cache, string name)

src/AzurePipelines/PipelineCachingCacheClient.cs

@@ -106,8 +106,9 @@ public PipelineCachingCacheClient(
         bool remoteCacheIsReadOnly,
         bool enableAsyncPublishing,
         bool enableAsyncMaterialization,
-        bool skipUnchangedOutputFiles)
-        : base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCAS, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles)
+        bool skipUnchangedOutputFiles,
+        bool touchOutputFiles)
+        : base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCAS, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles, touchOutputFiles)
     {
         _remoteCacheIsReadOnly = remoteCacheIsReadOnly;
         _universe = $"pccc-{(int)hasher.Info.HashType}-{InternalSeed}-" + (string.IsNullOrEmpty(universe) ? "DEFAULT" : universe);

src/Common/Caching/CacheClient.cs

@@ -40,6 +40,7 @@ public abstract class CacheClient : ICacheClient
     private readonly IContentHasher _hasher;
     private readonly IFingerprintFactory _fingerprintFactory;
     private readonly bool _enableAsyncMaterialization;
+    private readonly bool _touchOutputFiles;
     private readonly ICache _localCache;
     private readonly string _nugetPackageRoot;
     private readonly bool _canCloneInNugetCachePath;
@@ -57,7 +58,8 @@ protected CacheClient(
         int maxConcurrentCacheContentOperations,
         bool enableAsyncPublishing,
         bool enableAsyncMaterialization,
-        bool skipUnchangedOutputFiles)
+        bool skipUnchangedOutputFiles,
+        bool touchOutputFiles)
     {
         RootContext = rootContext;
         _fingerprintFactory = fingerprintFactory;
@@ -69,6 +71,7 @@ protected CacheClient(
         LocalCacheSession = localCas;
         EnableAsyncPublishing = enableAsyncPublishing;
         _enableAsyncMaterialization = enableAsyncMaterialization;
+        _touchOutputFiles = touchOutputFiles;
         GetFileRealizationMode = getFileRealizationMode;
 
         PutOrPlaceFileGate = new SemaphoreSlim(maxConcurrentCacheContentOperations);
@@ -494,6 +497,26 @@ async Task PlaceFilesAsync(CancellationToken ct)
             tasks.Add(placeFilesTask);
             await Task.WhenAll(tasks);
 
+            if (_touchOutputFiles)
+            {
+                // When touching files, use the same timestamp for every file to ensure we don't end up with some outputs with slightly different timestamps, which may lead to missed incrementality.
+                DateTime fileTimestamp = DateTime.Now;
+                foreach (KeyValuePair<string, ContentHash> kvp in nodeBuildResult.Outputs)
+                {
+                    string relativeFilePath = kvp.Key;
+
+                    // Avoid touching the reference assembly as incremental build functionality heavily depends on this file not being updated when it does not change.
+                    if (string.Equals(relativeFilePath, nodeContext.ReferenceAssemblyRelativePath, StringComparison.OrdinalIgnoreCase))
+                    {
+                        continue;
+                    }
+
+                    // Touch the file to ensure incremental builds understand that the file is up to date
+                    string absoluteFilePath = Path.Combine(RepoRoot, relativeFilePath);
+                    File.SetLastWriteTime(absoluteFilePath, fileTimestamp);
+                }
+            }
+
             if (_localCacheStateManager is not null)
             {
                 await _localCacheStateManager.WriteStateFileAsync(nodeContext, nodeBuildResult);

src/Common/Caching/CasCacheClient.cs

@@ -52,8 +52,9 @@ public CasCacheClient(
         int maxConcurrentCacheContentOperations,
         bool enableAsyncPublishing,
         bool enableAsyncMaterialization,
-        bool skipUnchangedOutputFiles)
-        : base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCacheSession, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles)
+        bool skipUnchangedOutputFiles,
+        bool touchOutputFiles)
+        : base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCacheSession, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles, touchOutputFiles)
     {
         ICacheSession cacheSession;
         if (remoteCache == null)

src/Common/Caching/LocalCacheStateManager.cs

@@ -86,27 +86,11 @@ internal async Task<List<KeyValuePair<string, ContentHash>>> GetOutOfDateFilesAs
         }
 
         List<KeyValuePair<string, ContentHash>> outOfDateFiles = new(nodeBuildResult.Outputs.Count);
-
-        // When touching files, use the same timestamp for every file to ensure we don't end up with some outputs with slightly different timestamps, which may lead to missed incrementality.
-        DateTime fileTimestamp = DateTime.Now;
-
         foreach (KeyValuePair<string, ContentHash> kvp in nodeBuildResult.Outputs)
         {
             string relativeFilePath = kvp.Key;
             ContentHash contentHash = kvp.Value;
-            if (IsFileUpToDate(context, depFile, relativeFilePath, contentHash))
-            {
-                // Avoid touching the reference assembly as incremental build functionality heavily depends on this file not being updated when it does not change.
-                if (string.Equals(relativeFilePath, nodeContext.ReferenceAssemblyRelativePath, StringComparison.OrdinalIgnoreCase))
-                {
-                    continue;
-                }
-
-                // Touch the file to ensure incremental builds understand that the file is up to date
-                string absoluteFilePath = Path.Combine(_repoRoot, relativeFilePath);
-                File.SetLastWriteTime(absoluteFilePath, fileTimestamp);
-            }
-            else
+            if (!IsFileUpToDate(context, depFile, relativeFilePath, contentHash))
             {
                 outOfDateFiles.Add(kvp);
             }

src/Common/PluginSettings.cs

@@ -109,6 +109,8 @@ public string LocalCacheRootPath
 
     public bool SkipUnchangedOutputFiles { get; init; }
 
+    public bool TouchOutputFiles { get; init; }
+
     public static T Create<T>(
         IReadOnlyDictionary<string, string> settings,
         PluginLoggerBase logger,

src/Common/build/Microsoft.MSBuildCache.Common.targets

@@ -24,6 +24,7 @@
     <MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheTargetsToIgnore</MSBuildCacheGlobalPropertiesToIgnore>
     <MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheIgnoreDotNetSdkPatchVersion</MSBuildCacheGlobalPropertiesToIgnore>
     <MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheSkipUnchangedOutputFiles</MSBuildCacheGlobalPropertiesToIgnore>
+    <MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheTouchOutputFiles</MSBuildCacheGlobalPropertiesToIgnore>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(MSBuildCacheEnabled)' != 'false'">
@@ -47,6 +48,7 @@
       <TargetsToIgnore>$(MSBuildCacheTargetsToIgnore)</TargetsToIgnore>
       <IgnoreDotNetSdkPatchVersion>$(MSBuildCacheIgnoreDotNetSdkPatchVersion)</IgnoreDotNetSdkPatchVersion>
       <SkipUnchangedOutputFiles>$(MSBuildCacheSkipUnchangedOutputFiles)</SkipUnchangedOutputFiles>
+      <TouchOutputFiles>$(MSBuildCacheTouchOutputFiles)</TouchOutputFiles>
     </ProjectCachePlugin>
   </ItemGroup>
 

src/Local/MSBuildCacheLocalPlugin.cs

@@ -63,6 +63,7 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
             Settings.MaxConcurrentCacheContentOperations,
             Settings.AsyncCachePublishing,
             Settings.AsyncCacheMaterialization,
-            Settings.SkipUnchangedOutputFiles);
+            Settings.SkipUnchangedOutputFiles,
+            Settings.TouchOutputFiles);
     }
 }