Merge pull request dotnet#5546 from cdmihai/iSystemData

- Made IFileSystem public as MSBuildFileSystemBase. This interface will also be passed to the future project cache interface so that caches can reuse IO from evaluation and between themselves.
- EvaluationContext can take an MSBuildFileSystemBase. This enables QuickBuild and VisualStudio to share their file system caches with msbuild evaluations.

ref/Microsoft.Build/net/Microsoft.Build.cs

@@ -869,6 +869,7 @@ public partial class EvaluationContext
     {
         internal EvaluationContext() { }
         public static Microsoft.Build.Evaluation.Context.EvaluationContext Create(Microsoft.Build.Evaluation.Context.EvaluationContext.SharingPolicy policy) { throw null; }
+        public static Microsoft.Build.Evaluation.Context.EvaluationContext Create(Microsoft.Build.Evaluation.Context.EvaluationContext.SharingPolicy policy, Microsoft.Build.FileSystem.MSBuildFileSystemBase fileSystem) { throw null; }
         public enum SharingPolicy
         {
             Isolated = 1,
@@ -1402,6 +1403,25 @@ public enum TargetResultCode : byte
         Success = (byte)1,
     }
 }
+namespace Microsoft.Build.FileSystem
+{
+    public abstract partial class MSBuildFileSystemBase
+    {
+        protected MSBuildFileSystemBase() { }
+        public abstract bool DirectoryExists(string path);
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateDirectories(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateFiles(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateFileSystemEntries(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract bool FileExists(string path);
+        public abstract bool FileOrDirectoryExists(string path);
+        public abstract System.IO.FileAttributes GetAttributes(string path);
+        public abstract System.IO.Stream GetFileStream(string path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share);
+        public abstract System.DateTime GetLastWriteTimeUtc(string path);
+        public abstract System.IO.TextReader ReadFile(string path);
+        public abstract byte[] ReadFileAllBytes(string path);
+        public abstract string ReadFileAllText(string path);
+    }
+}
 namespace Microsoft.Build.Globbing
 {
     public partial class CompositeGlob : Microsoft.Build.Globbing.IMSBuildGlob

ref/Microsoft.Build/netstandard/Microsoft.Build.cs

@@ -869,6 +869,7 @@ public partial class EvaluationContext
     {
         internal EvaluationContext() { }
         public static Microsoft.Build.Evaluation.Context.EvaluationContext Create(Microsoft.Build.Evaluation.Context.EvaluationContext.SharingPolicy policy) { throw null; }
+        public static Microsoft.Build.Evaluation.Context.EvaluationContext Create(Microsoft.Build.Evaluation.Context.EvaluationContext.SharingPolicy policy, Microsoft.Build.FileSystem.MSBuildFileSystemBase fileSystem) { throw null; }
         public enum SharingPolicy
         {
             Isolated = 1,
@@ -1396,6 +1397,25 @@ public enum TargetResultCode : byte
         Success = (byte)1,
     }
 }
+namespace Microsoft.Build.FileSystem
+{
+    public abstract partial class MSBuildFileSystemBase
+    {
+        protected MSBuildFileSystemBase() { }
+        public abstract bool DirectoryExists(string path);
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateDirectories(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateFiles(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract System.Collections.Generic.IEnumerable<string> EnumerateFileSystemEntries(string path, string searchPattern="*", System.IO.SearchOption searchOption=(System.IO.SearchOption)(0));
+        public abstract bool FileExists(string path);
+        public abstract bool FileOrDirectoryExists(string path);
+        public abstract System.IO.FileAttributes GetAttributes(string path);
+        public abstract System.IO.Stream GetFileStream(string path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share);
+        public abstract System.DateTime GetLastWriteTimeUtc(string path);
+        public abstract System.IO.TextReader ReadFile(string path);
+        public abstract byte[] ReadFileAllBytes(string path);
+        public abstract string ReadFileAllText(string path);
+    }
+}
 namespace Microsoft.Build.Globbing
 {
     public partial class CompositeGlob : Microsoft.Build.Globbing.IMSBuildGlob

src/Build.UnitTests/Definition/ProjectEvaluationContext_Tests.cs

@@ -86,6 +86,51 @@ public void SharedContextShouldGetReusedWhereasIsolatedContextShouldNot(Evaluati
             }
         }
 
+        [Fact]
+        public void PassedInFileSystemShouldBeReusedInSharedContext()
+        {
+            var projectFiles = new[]
+            {
+                _env.CreateFile("1.proj", @"<Project> <PropertyGroup Condition=`Exists('1.file')`></PropertyGroup> </Project>".Cleanup()).Path,
+                _env.CreateFile("2.proj", @"<Project> <PropertyGroup Condition=`Exists('2.file')`></PropertyGroup> </Project>".Cleanup()).Path
+            };
+
+            var projectCollection = _env.CreateProjectCollection().Collection;
+            var fileSystem = new Helpers.LoggingFileSystem();
+            var evaluationContext = EvaluationContext.Create(EvaluationContext.SharingPolicy.Shared, fileSystem);
+
+            foreach (var projectFile in projectFiles)
+            {
+                Project.FromFile(
+                    projectFile,
+                    new ProjectOptions
+                    {
+                        ProjectCollection = projectCollection,
+                        EvaluationContext = evaluationContext
+                    }
+                );
+            }
+
+            fileSystem.ExistenceChecks.OrderBy(kvp => kvp.Key)
+                .ShouldBe(
+                    new Dictionary<string, int>
+                    {
+                        {Path.Combine(_env.DefaultTestDirectory.Path, "1.file"), 1},
+                        {Path.Combine(_env.DefaultTestDirectory.Path, "2.file"), 1}
+                    }.OrderBy(kvp => kvp.Key));
+
+            fileSystem.DirectoryEntryExistsCalls.ShouldBe(2);
+        }
+
+        [Fact]
+        public void IsolatedContextShouldNotSupportBeingPassedAFileSystem()
+        {
+            _env.DoNotLaunchDebugger();
+
+            var fileSystem = new Helpers.LoggingFileSystem();
+            Should.Throw<ArgumentException>(() => EvaluationContext.Create(EvaluationContext.SharingPolicy.Isolated, fileSystem));
+        }
+
         [Theory]
         [InlineData(EvaluationContext.SharingPolicy.Shared)]
         [InlineData(EvaluationContext.SharingPolicy.Isolated)]

src/Build/Evaluation/Context/EvaluationContext.cs

@@ -6,6 +6,7 @@
 using System.Collections.Immutable;
 using System.Threading;
 using Microsoft.Build.BackEnd.SdkResolution;
+using Microsoft.Build.FileSystem;
 using Microsoft.Build.Internal;
 using Microsoft.Build.Shared;
 using Microsoft.Build.Shared.FileSystem;
@@ -17,13 +18,20 @@ namespace Microsoft.Build.Evaluation.Context
     ///     evaluations).
     ///     The caller should throw away the context when the environment changes (IO, environment variables, SDK resolution
     ///     inputs, etc).
-    ///     This class and it's closure needs to be thread safe since API users can do evaluations in parallel.
+    ///     This class and its closure needs to be thread safe since API users can do evaluations in parallel.
     /// </summary>
     public class EvaluationContext
     {
         public enum SharingPolicy
         {
+            /// <summary>
+            /// Instructs the <see cref="EvaluationContext"/> to reuse state between the different project evaluations that use it.
+            /// </summary>
             Shared,
+
+            /// <summary>
+            /// Instructs the <see cref="EvaluationContext"/> not to reuse state between the different project evaluations that use it.
+            /// </summary>
             Isolated
         }
 
@@ -40,15 +48,21 @@ public enum SharingPolicy
         /// <summary>
         /// Key to file entry list. Example usages: cache glob expansion and intermediary directory expansions during glob expansion.
         /// </summary>
-        internal ConcurrentDictionary<string, ImmutableArray<string>> FileEntryExpansionCache { get; }
+        private ConcurrentDictionary<string, ImmutableArray<string>> FileEntryExpansionCache { get; }
 
-        internal EvaluationContext(SharingPolicy policy)
+        private EvaluationContext(SharingPolicy policy, IFileSystem fileSystem)
         {
+            // Unsupported case: isolated context with non null file system.
+            // Isolated means caches aren't reused, but the given file system might cache.
+            ErrorUtilities.VerifyThrowArgument(
+                policy == SharingPolicy.Shared || fileSystem == null,
+                "IsolatedContextDoesNotSupportFileSystem");
+
             Policy = policy;
 
             SdkResolverService = new CachingSdkResolverService();
             FileEntryExpansionCache = new ConcurrentDictionary<string, ImmutableArray<string>>();
-            FileSystem = new CachingFileSystemWrapper(FileSystems.Default);
+            FileSystem = fileSystem ?? new CachingFileSystemWrapper(FileSystems.Default);
             EngineFileUtilities = new EngineFileUtilities(new FileMatcher(FileSystem, FileEntryExpansionCache));
         }
 
@@ -57,7 +71,28 @@ internal EvaluationContext(SharingPolicy policy)
         /// </summary>
         public static EvaluationContext Create(SharingPolicy policy)
         {
-            var context = new EvaluationContext(policy);
+
+            // ReSharper disable once IntroduceOptionalParameters.Global
+            // do not remove this method to avoid breaking binary compatibility
+            return Create(policy, fileSystem: null);
+        }
+
+        /// <summary>
+        ///     Factory for <see cref="EvaluationContext" />
+        /// </summary>
+        /// <param name="policy"> The <see cref="SharingPolicy"/> to use.</param>
+        /// <param name="fileSystem">The <see cref="IFileSystem"/> to use.
+        ///     This parameter is compatible only with <see cref="SharingPolicy.Shared"/>.
+        ///     The method throws if a file system is used with <see cref="SharingPolicy.Isolated"/>.
+        ///     The reasoning is that <see cref="SharingPolicy.Isolated"/> means not reusing any caches between evaluations,
+        ///     and the passed in <paramref name="fileSystem"/> might cache state.
+        /// </param>
+        public static EvaluationContext Create(SharingPolicy policy, MSBuildFileSystemBase fileSystem)
+        {
+            var context = new EvaluationContext(
+                policy,
+                fileSystem == null ? null : new MSBuildFileSystemAdapter(fileSystem));
+
             TestOnlyHookOnCreate?.Invoke(context);
 
             return context;

src/Build/FileSystem/MSBuildFileSystemAdapter.cs

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.Build.Shared.FileSystem;
+
+namespace Microsoft.Build.FileSystem
+{
+     internal class MSBuildFileSystemAdapter : IFileSystem
+    {
+        private readonly MSBuildFileSystemBase _msbuildFileSystem;
+        public MSBuildFileSystemAdapter(MSBuildFileSystemBase msbuildFileSystem)
+        {
+            _msbuildFileSystem = msbuildFileSystem;
+        }
+        public TextReader ReadFile(string path) => _msbuildFileSystem.ReadFile(path);
+
+        public Stream GetFileStream(string path, FileMode mode, FileAccess access, FileShare share) => _msbuildFileSystem.GetFileStream(path, mode, access, share);
+
+        public string ReadFileAllText(string path) => _msbuildFileSystem.ReadFileAllText(path);
+
+        public byte[] ReadFileAllBytes(string path) => _msbuildFileSystem.ReadFileAllBytes(path);
+
+        public IEnumerable<string> EnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            return _msbuildFileSystem.EnumerateFiles(path, searchPattern, searchOption);
+        }
+
+        public IEnumerable<string> EnumerateDirectories(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            return _msbuildFileSystem.EnumerateDirectories(path, searchPattern, searchOption);
+        }
+
+        public IEnumerable<string> EnumerateFileSystemEntries(
+            string path,
+            string searchPattern = "*",
+            SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            return _msbuildFileSystem.EnumerateFileSystemEntries(path, searchPattern, searchOption);
+        }
+
+        public FileAttributes GetAttributes(string path) => _msbuildFileSystem.GetAttributes(path);
+
+        public DateTime GetLastWriteTimeUtc(string path) => _msbuildFileSystem.GetLastWriteTimeUtc(path);
+
+        public bool DirectoryExists(string path) => _msbuildFileSystem.DirectoryExists(path);
+
+        public bool FileExists(string path) => _msbuildFileSystem.FileExists(path);
+
+        public bool DirectoryEntryExists(string path) => _msbuildFileSystem.FileOrDirectoryExists(path);
+    }
+}

src/Build/FileSystem/MSBuildFileSystemBase.cs

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Microsoft.Build.FileSystem
+{
+    /// <summary>
+    /// Abstracts away some file system operations.
+    ///
+    /// Implementations:
+    /// - must be thread safe
+    /// - may cache some or all the calls.
+    /// </summary>
+    public abstract class MSBuildFileSystemBase
+    {
+        /// <summary>
+        /// Use this for var sr = new StreamReader(path)
+        /// </summary>
+        public abstract TextReader ReadFile(string path);
+
+        /// <summary>
+        /// Use this for new FileStream(path, mode, access, share)
+        /// </summary>
+        public abstract Stream GetFileStream(string path, FileMode mode, FileAccess access, FileShare share);
+
+        /// <summary>
+        /// Use this for File.ReadAllText(path)
+        /// </summary>
+        public abstract string ReadFileAllText(string path);
+
+        /// <summary>
+        /// Use this for File.ReadAllBytes(path)
+        /// </summary>
+        public abstract byte[] ReadFileAllBytes(string path);
+
+        /// <summary>
+        /// Use this for Directory.EnumerateFiles(path, pattern, option)
+        /// </summary>
+        public abstract IEnumerable<string> EnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly);
+
+        /// <summary>
+        /// Use this for Directory.EnumerateFolders(path, pattern, option)
+        /// </summary>
+        public abstract IEnumerable<string> EnumerateDirectories(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly);
+
+        /// <summary>
+        /// Use this for Directory.EnumerateFileSystemEntries(path, pattern, option)
+        /// </summary>
+        public abstract IEnumerable<string> EnumerateFileSystemEntries(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly);
+
+        /// <summary>
+        /// Use this for File.GetAttributes()
+        /// </summary>
+        public abstract FileAttributes GetAttributes(string path);
+
+        /// <summary>
+        /// Use this for File.GetLastWriteTimeUtc(path)
+        /// </summary>
+        public abstract DateTime GetLastWriteTimeUtc(string path);
+
+        /// <summary>
+        /// Use this for Directory.Exists(path)
+        /// </summary>
+        public abstract bool DirectoryExists(string path);
+
+        /// <summary>
+        /// Use this for File.Exists(path)
+        /// </summary>
+        public abstract bool FileExists(string path);
+
+        /// <summary>
+        /// Use this for File.Exists(path) || Directory.Exists(path)
+        /// </summary>
+        public abstract bool FileOrDirectoryExists(string path);
+    }
+}

src/Build/Microsoft.Build.csproj

@@ -157,6 +157,7 @@
     <Compile Include="BackEnd\Components\Caching\ConfigCacheWithOverride.cs" />
     <Compile Include="BackEnd\Components\Caching\ResultsCacheWithOverride.cs" />
     <Compile Include="BackEnd\Components\SdkResolution\TranslationHelpers.cs" />
+    <Compile Include="FileSystem\MSBuildFileSystemBase.cs" />
     <Compile Include="Utilities\NuGetFrameworkWrapper.cs" />
     <Compile Include="ObjectModelRemoting\ConstructionObjectLinks\ProjectUsingTaskParameterElementLink.cs" />
     <Compile Include="ObjectModelRemoting\ExternalProjectsProvider.cs" />
@@ -263,6 +264,7 @@
     <Compile Include="Evaluation\LazyItemEvaluator.ItemFactoryWrapper.cs" />
     <Compile Include="Evaluation\LazyItemEvaluator.RemoveOperation.cs" />
     <Compile Include="Evaluation\MetadataReference.cs" />
+    <Compile Include="FileSystem\MSBuildFileSystemAdapter.cs" />
     <Compile Include="Graph\ProjectGraphEntryPoint.cs" />
     <Compile Include="Graph\ProjectGraph.cs" />
     <Compile Include="Graph\ProjectGraphNode.cs" />

src/Build/Resources/Strings.resx

@@ -1831,4 +1831,7 @@ Utilization:          {0} Average Utilization: {1:###.0}</value>
   <data name="StaticGraphConstructionMetrics" xml:space="preserve">
     <value>"Static graph loaded in {0} seconds: {1} nodes, {2} edges"</value>
   </data>
-</root>
+  <data name="IsolatedContextDoesNotSupportFileSystem" xml:space="preserve">
+    <value>"EvaluationContext objects created with SharingPolicy.Isolated do not support being passed an MSBuildFileSystemBase file system."</value>
+  </data>
+</root>