Merge pull request #8726 from dfederm/project-cache-vnext

This change add "cache add" functionality to project caching.

Today project caching exposes a hook to plugins for "I'm about to build this thing, do you want to take over instead?" and that's it. This change adds a few more hooks which report file accesses and processes (via Detours) and also for when a project finishes building. This allows a plugin to collect the file accesses and add entries to the cache for future replayability.

Author: rainersigwald

NuGet.config

@@ -8,6 +8,7 @@
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
     <add key="dotnet8" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet8/nuget/v3/index.json" />
     <add key="dotnet8-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8-transport/nuget/v3/index.json" />
+    <add key="BuildXL" value="https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources />
 </configuration>

eng/dependabot/Packages.props

@@ -19,6 +19,9 @@
     <PackageVersion Include="LargeAddressAware" Version="1.0.5" />
     <PackageVersion Update="LargeAddressAware" Condition="'$(LargeAddressAwareVersion)' != ''" Version="$(LargeAddressAwareVersion)" />
 
+    <PackageVersion Include="Microsoft.BuildXL.Processes" Version="0.1.0-20230727.4.2" />
+    <PackageVersion Update="Microsoft.BuildXL.Processes" Condition="'$(BuildXLProcessesVersion)' != ''" Version="$(BuildXLProcessesVersion)" />
+
     <PackageVersion Include="Microsoft.VisualStudio.Setup.Configuration.Interop" Version="3.2.2146" PrivateAssets="All" />
     <PackageVersion Update="Microsoft.VisualStudio.Setup.Configuration.Interop" Condition="'$(MicrosoftVisualStudioSetupConfigurationInteropVersion)' != ''" Version="$(MicrosoftVisualStudioSetupConfigurationInteropVersion)" PrivateAssets="All" />
 

src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using Microsoft.Build.BackEnd;
 using Microsoft.Build.Framework;
+using Microsoft.Build.Framework.FileAccess;
 using Microsoft.Build.Shared;
 using Microsoft.Build.Utilities;
 using Xunit;
@@ -25,21 +26,67 @@ public class TaskHostTaskComplete_Tests
         [Fact]
         public void TestConstructors()
         {
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null);
-            TaskHostTaskComplete complete2 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), null);
-            TaskHostTaskComplete complete3 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, new ArgumentOutOfRangeException()), null);
-            TaskHostTaskComplete complete4 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), null);
+#if FEATURE_REPORTFILEACCESSES
+            var fileAccessData = new List<FileAccessData>()
+            {
+                new FileAccessData(
+                    ReportedFileOperation.CreateFile,
+                    RequestedAccess.Read,
+                    0,
+                    0,
+                    DesiredAccess.GENERIC_READ,
+                    FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
+                    "foo",
+                    null,
+                    true),
+            };
+#endif
+
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success),
+#if FEATURE_REPORTFILEACCESSES
+                fileAccessData,
+#endif
+                null);
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure),
+#if FEATURE_REPORTFILEACCESSES
+                fileAccessData,
+#endif
+                null);
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization,
+                new ArgumentOutOfRangeException()),
+#if FEATURE_REPORTFILEACCESSES
+                fileAccessData,
+#endif
+                null);
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()),
+#if FEATURE_REPORTFILEACCESSES
+                fileAccessData,
+#endif
+                null);
 
             IDictionary<string, object> parameters = new Dictionary<string, object>();
-            TaskHostTaskComplete complete5 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             IDictionary<string, object> parameters2 = new Dictionary<string, object>();
             parameters2.Add("Text", "Hello!");
             parameters2.Add("MyBoolValue", true);
             parameters2.Add("MyITaskItem", new TaskItem("ABC"));
             parameters2.Add("ItemArray", new ITaskItem[] { new TaskItem("DEF"), new TaskItem("GHI"), new TaskItem("JKL") });
-
-            TaskHostTaskComplete complete6 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), null);
+            _ = new TaskHostTaskComplete(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
         }
 
         /// <summary>
@@ -60,7 +107,12 @@ public void TestInvalidConstructors()
         [Fact]
         public void TestTranslationWithNullDictionary()
         {
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null);
+            TaskHostTaskComplete complete = new(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
             INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
@@ -78,7 +130,12 @@ public void TestTranslationWithNullDictionary()
         [Fact]
         public void TestTranslationWithEmptyDictionary()
         {
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary<string, object>()), null);
+            TaskHostTaskComplete complete = new(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary<string, object>()),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
             INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
@@ -99,7 +156,12 @@ public void TestTranslationWithValueTypesInDictionary()
             IDictionary<string, object> parameters = new Dictionary<string, object>();
             parameters.Add("Text", "Foo");
             parameters.Add("BoolValue", false);
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
+            TaskHostTaskComplete complete = new(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
             INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
@@ -121,7 +183,12 @@ public void TestTranslationWithITaskItemInDictionary()
         {
             IDictionary<string, object> parameters = new Dictionary<string, object>();
             parameters.Add("TaskItemValue", new TaskItem("Foo"));
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
+            TaskHostTaskComplete complete = new(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
             INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
@@ -142,7 +209,12 @@ public void TestTranslationWithITaskItemArrayInDictionary()
         {
             IDictionary<string, object> parameters = new Dictionary<string, object>();
             parameters.Add("TaskItemArrayValue", new ITaskItem[] { new TaskItem("Foo"), new TaskItem("Baz") });
-            TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
+            TaskHostTaskComplete complete = new(
+                new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
+#if FEATURE_REPORTFILEACCESSES
+                null,
+#endif
+                null);
 
             ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
             INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
@@ -168,7 +240,12 @@ private void AssertInvalidConstructorThrows(Type expectedExceptionType, TaskComp
 
             try
             {
-                TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), buildProcessEnvironment);
+                TaskHostTaskComplete complete = new(
+                    new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs),
+#if FEATURE_REPORTFILEACCESSES
+                    null,
+#endif
+                    buildProcessEnvironment);
             }
             catch (Exception e)
             {

src/Build/BackEnd/BuildManager/BuildManager.cs

@@ -25,6 +25,7 @@
 using Microsoft.Build.Exceptions;
 using Microsoft.Build.Experimental;
 using Microsoft.Build.Experimental.ProjectCache;
+using Microsoft.Build.FileAccesses;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Framework.Telemetry;
 using Microsoft.Build.Graph;
@@ -558,6 +559,19 @@ public void BeginBuild(BuildParameters parameters)
                     _buildParameters.OutputResultsCacheFile = FileUtilities.NormalizePath("msbuild-cache");
                 }
 
+#if FEATURE_REPORTFILEACCESSES
+                if (_buildParameters.ReportFileAccesses)
+                {
+                    // To properly report file access, we need to disable the in-proc node which won't be detoured.
+                    _buildParameters.DisableInProcNode = true;
+
+                    // Node reuse must be disabled as future builds will not be able to listen to events raised by detours.
+                    _buildParameters.EnableNodeReuse = false;
+
+                    _componentFactories.ReplaceFactory(BuildComponentType.NodeLauncher, DetouredNodeLauncher.CreateComponent);
+                }
+#endif
+
                 // Initialize components.
                 _nodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.NodeManager) as INodeManager;
 
@@ -572,9 +586,17 @@ public void BeginBuild(BuildParameters parameters)
 
                 InitializeCaches();
 
+#if FEATURE_REPORTFILEACCESSES
+                var fileAccessManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager) as IFileAccessManager;
+#endif
+
                 _projectCacheService = new ProjectCacheService(
                     this,
                     loggingService,
+#if FEATURE_REPORTFILEACCESSES
+                    fileAccessManager,
+#endif
+                    _configCache,
                     _buildParameters.ProjectCacheDescriptor);
 
                 _taskHostNodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.TaskHostNodeManager) as INodeManager;
@@ -584,7 +606,9 @@ public void BeginBuild(BuildParameters parameters)
                 _nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfiguration, BuildRequestConfiguration.FactoryForDeserialization, this);
                 _nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfigurationResponse, BuildRequestConfigurationResponse.FactoryForDeserialization, this);
                 _nodeManager.RegisterPacketHandler(NodePacketType.BuildResult, BuildResult.FactoryForDeserialization, this);
+                _nodeManager.RegisterPacketHandler(NodePacketType.FileAccessReport, FileAccessReport.FactoryForDeserialization, this);
                 _nodeManager.RegisterPacketHandler(NodePacketType.NodeShutdown, NodeShutdown.FactoryForDeserialization, this);
+                _nodeManager.RegisterPacketHandler(NodePacketType.ProcessReport, ProcessReport.FactoryForDeserialization, this);
                 _nodeManager.RegisterPacketHandler(NodePacketType.ResolveSdkRequest, SdkResolverRequest.FactoryForDeserialization, SdkResolverService as INodePacketHandler);
                 _nodeManager.RegisterPacketHandler(NodePacketType.ResourceRequest, ResourceRequest.FactoryForDeserialization, this);
 
@@ -1564,6 +1588,16 @@ private void ProcessPacket(int node, INodePacket packet)
                         HandleNodeShutdown(node, shutdownPacket);
                         break;
 
+                    case NodePacketType.FileAccessReport:
+                        FileAccessReport fileAccessReport = ExpectPacketType<FileAccessReport>(packet, NodePacketType.FileAccessReport);
+                        HandleFileAccessReport(node, fileAccessReport);
+                        break;
+
+                    case NodePacketType.ProcessReport:
+                        ProcessReport processReport = ExpectPacketType<ProcessReport>(packet, NodePacketType.ProcessReport);
+                        HandleProcessReport(node, processReport);
+                        break;
+
                     default:
                         ErrorUtilities.ThrowInternalError("Unexpected packet received by BuildManager: {0}", packet.Type);
                         break;
@@ -2371,6 +2405,39 @@ private void HandleResult(int node, BuildResult result)
                 configuration.ProjectTargets ??= result.ProjectTargets;
             }
 
+            // Only report results to the project cache services if it's the result for a build submission.
+            // Note that graph builds create a submission for each node in the graph, so each node in the graph will be
+            // handled here. This intentionally mirrors the behavior for cache requests, as it doesn't make sense to
+            // report for projects which aren't going to be requested. Ideally, *any* request could be handled, but that
+            // would require moving the cache service interactions to the Scheduler.
+            if (_buildSubmissions.TryGetValue(result.SubmissionId, out BuildSubmission buildSubmission))
+            {
+                // The result may be associated with the build submission due to it being the submission which
+                // caused the build, but not the actual request which was originally used with the build submission.
+                // ie. it may be a dependency of the "root-level" project which is associated with this submission, which
+                // isn't what we're looking for. Ensure only the actual submission's request is considered.
+                if (buildSubmission.BuildRequest != null
+                    && buildSubmission.BuildRequest.ConfigurationId == configuration.ConfigurationId
+                    && _projectCacheService.ShouldUseCache(configuration))
+                {
+                    BuildEventContext buildEventContext = _projectStartedEvents.TryGetValue(result.SubmissionId, out BuildEventArgs buildEventArgs)
+                        ? buildEventArgs.BuildEventContext
+                        : new BuildEventContext(result.SubmissionId, node, configuration.Project?.EvaluationId ?? BuildEventContext.InvalidEvaluationId, configuration.ConfigurationId, BuildEventContext.InvalidProjectContextId, BuildEventContext.InvalidTargetId, BuildEventContext.InvalidTaskId);
+                    try
+                    {
+                        _projectCacheService.HandleBuildResultAsync(configuration, result, buildEventContext, _executionCancellationTokenSource.Token).Wait();
+                    }
+                    catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException))
+                    {
+                        // The build is being cancelled. Swallow any exceptions related specifically to cancellation.
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        // The build is being cancelled. Swallow any exceptions related specifically to cancellation.
+                    }
+                }
+            }
+
             IEnumerable<ScheduleResponse> response = _scheduler.ReportResult(node, result);
             PerformSchedulingActions(response);
         }
@@ -2437,6 +2504,36 @@ private void HandleNodeShutdown(int node, NodeShutdown shutdownPacket)
             CheckForActiveNodesAndCleanUpSubmissions();
         }
 
+        /// <summary>
+        /// Report the received <paramref name="fileAccessReport"/> to the file access manager.
+        /// </summary>
+        /// <param name="nodeId">The id of the node from which the <paramref name="fileAccessReport"/> was received.</param>
+        /// <param name="fileAccessReport">The file access report.</param>
+        private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport)
+        {
+#if FEATURE_REPORTFILEACCESSES
+            if (_buildParameters.ReportFileAccesses)
+            {
+                ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessReport.FileAccessData, nodeId);
+            }
+#endif
+        }
+
+        /// <summary>
+        /// Report the received <paramref name="processReport"/> to the file access manager.
+        /// </summary>
+        /// <param name="nodeId">The id of the node from which the <paramref name="processReport"/> was received.</param>
+        /// <param name="processReport">The process data report.</param>
+        private void HandleProcessReport(int nodeId, ProcessReport processReport)
+        {
+#if FEATURE_REPORTFILEACCESSES
+            if (_buildParameters.ReportFileAccesses)
+            {
+                ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportProcess(processReport.ProcessData, nodeId);
+            }
+#endif
+        }
+
         /// <summary>
         /// If there are no more active nodes, cleans up any remaining submissions.
         /// </summary>