Added fsx evaluation/running to new testing framework. (dotnet#10106)

* Added fsx evaluation via the FSharpScript.Eval as well as FsiEvaluationSession.EvalInteractionNonThrowing

* Turn off test filter for XUnit, and add to NUnit

Author: vzarytovskii

eng/Build.ps1

@@ -286,11 +286,11 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
 }
 
 function TestUsingXUnit([string] $testProject, [string] $targetFramework, [string]$testadapterpath) {
-    TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $false
+    TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $true
 }
 
 function TestUsingNUnit([string] $testProject, [string] $targetFramework, [string]$testadapterpath) {
-    TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $true
+    TestUsingMsBuild -testProject $testProject -targetFramework $targetFramework -testadapterpath $testadapterpath -noTestFilter $false
 }
 
 function BuildCompiler() {

tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj

@@ -48,6 +48,7 @@
     <Compile Include="ConstraintSolver\MemberConstraints.fs" />
     <Compile Include="Interop\SimpleInteropTests.fs" />
     <Compile Include="Interop\VisibilityTests.fs" />
+    <Compile Include="Scripting\Interactive.fs" />
   </ItemGroup>
 
   <ItemGroup>

tests/FSharp.Compiler.ComponentTests/Language/CompilerDirectiveTests.fs

@@ -23,3 +23,13 @@ module ``Test Compiler Directives`` =
         """ |> compile
             |> shouldFail
             |> withSingleDiagnostic (Warning 213, Line 2, Col 1, Line 2, Col 10, "'' is not a valid assembly name")
+
+module ``Test compiler directives in FSI`` =
+    [<Fact>]
+    let ``r# "" is invalid`` () =
+        Fsx"""
+#r ""
+        """ |> ignoreWarnings
+            |> eval
+            |> shouldFail
+            |> withSingleDiagnostic (Error 2301, Line 2, Col 1, Line 2, Col 6, "'' is not a valid assembly name")

tests/FSharp.Compiler.ComponentTests/Scripting/Interactive.fs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+namespace FSharp.Compiler.ComponentTests.Scripting
+
+open Xunit
+open FSharp.Test.Utilities.Compiler
+
+module ``Interactive tests`` =
+    [<Fact>]
+    let ``Eval object value``() =
+        Fsx "1+1"
+        |> eval
+        |> shouldSucceed
+        |> withEvalTypeEquals typeof<int>
+        |> withEvalValueEquals 2
+
+
+module ``External FSI tests`` =
+    [<Fact>]
+    let ``Eval object value``() =
+        Fsx "1+1"
+        |> runFsi
+        |> shouldSucceed
+
+    [<Fact>]
+    let ``Invalid expression should fail``() =
+            Fsx "1+a"
+            |> runFsi
+            |> shouldFail

tests/FSharp.Test.Utilities/Compiler.fs

@@ -2,6 +2,8 @@
 
 namespace FSharp.Test.Utilities
 
+open FSharp.Compiler.Interactive.Shell
+open FSharp.Compiler.Scripting
 open FSharp.Compiler.SourceCodeServices
 open FSharp.Test.Utilities
 open FSharp.Test.Utilities.Assert
@@ -68,17 +70,23 @@ module rec Compiler =
           Range:   Range
           Message: string }
 
+    type EvalOutput = Result<FsiValue option, exn>
+
     type ExecutionOutput =
         { ExitCode: int
           StdOut:   string
           StdErr:   string }
 
+    type RunOutput =
+        | EvalOutput of EvalOutput
+        | ExecutionOutput of ExecutionOutput
+
     type Output =
         { OutputPath:   string option
           Dependencies: string list
           Adjust:       int
           Diagnostics:  ErrorInfo list
-          Output:       ExecutionOutput option }
+          Output:       RunOutput option }
 
     type TestResult =
         | Success of Output
@@ -400,15 +408,71 @@ module rec Compiler =
             | None -> failwith "Compilation didn't produce any output. Unable to run. (did you forget to set output type to Exe?)"
             | Some p ->
                 let (exitCode, output, errors) = CompilerAssert.ExecuteAndReturnResult (p, s.Dependencies, false)
-                let executionResult = { s with Output = Some { ExitCode = exitCode; StdOut = output; StdErr = errors } }
+                let executionResult = { s with Output = Some (ExecutionOutput { ExitCode = exitCode; StdOut = output; StdErr = errors }) }
                 if exitCode = 0 then
                     Success executionResult
                 else
                     Failure executionResult
 
     let compileAndRun = compile >> run
-
     let compileExeAndRun = asExe >> compileAndRun
+    let private evalFSharp (fs: FSharpCompilationSource) : TestResult =
+        let source = getSource fs.Source
+        let options = fs.Options |> Array.ofList
+
+        use script = new FSharpScript(additionalArgs=options)
+
+        let ((evalresult: Result<FsiValue option, exn>), (err: FSharpErrorInfo[])) = script.Eval(source)
+
+        let diagnostics = err |> fromFSharpErrorInfo
+
+        let result =
+            { OutputPath   = None
+              Dependencies = []
+              Adjust       = 0
+              Diagnostics  = diagnostics
+              Output       = Some(EvalOutput evalresult) }
+
+        let (errors, warnings) = partitionErrors diagnostics
+
+        let evalError = match evalresult with Ok _ -> false | _ -> true
+
+        if evalError || errors.Length > 0 || (warnings.Length > 0 && not fs.IgnoreWarnings) then
+            Failure result
+        else
+            Success result
+
+    let eval (cUnit: CompilationUnit) : TestResult =
+        match cUnit with
+        | FS fs -> evalFSharp fs
+        | _ -> failwith "Script evaluation is only supported for F#."
+
+    let runFsi (cUnit: CompilationUnit) : TestResult =
+        match cUnit with
+        | FS fs ->
+            let source = getSource fs.Source
+
+            let options = fs.Options |> Array.ofList
+
+            let errors = CompilerAssert.RunScriptWithOptionsAndReturnResult options source
+
+            let result =
+                { OutputPath   = None
+                  Dependencies = []
+                  Adjust       = 0
+                  Diagnostics  = []
+                  Output       = None }
+
+            if errors.Count > 0 then
+                let output = ExecutionOutput {
+                    ExitCode = -1
+                    StdOut   = String.Empty
+                    StdErr   = ((errors |> String.concat "\n").Replace("\r\n","\n")) }
+                Failure { result with Output = Some output }
+            else
+                Success result
+        | _ -> failwith "FSI running only supports F#."
+
 
     let private createBaselineErrors (baseline: Baseline) actualErrors extension : unit =
         match baseline.SourceFilename with
@@ -465,6 +529,7 @@ module rec Compiler =
                     if not success then
                         createBaselineErrors bsl actualIL "fs.il.err"
                         Assert.Fail(errorMsg)
+
     let verifyILBaseline (cUnit: CompilationUnit) : CompilationUnit =
         match cUnit with
         | FS fs ->
@@ -541,7 +606,7 @@ module rec Compiler =
 
         let private assertResultsCategory (what: string) (selector: Output -> ErrorInfo list) (expected: ErrorInfo list) (result: TestResult) : TestResult =
             match result with
-             | Success r | Failure r ->
+            | Success r | Failure r ->
                 assertErrors what r.Adjust (selector r) expected
             result
 
@@ -605,7 +670,7 @@ module rec Compiler =
             result
 
         let withMessages (messages: string list) (result: TestResult) : TestResult =
-             checkErrorMessages messages (fun r -> r.Diagnostics) result
+            checkErrorMessages messages (fun r -> r.Diagnostics) result
 
         let withMessage (message: string) (result: TestResult) : TestResult =
             withMessages [message] result
@@ -627,7 +692,10 @@ module rec Compiler =
             | Success r | Failure r ->
                 match r.Output with
                 | None -> failwith "Execution output is missing, cannot check exit code."
-                | Some o -> Assert.AreEqual(o.ExitCode, expectedExitCode, sprintf "Exit code was expected to be: %A, but got %A." expectedExitCode o.ExitCode)
+                | Some o ->
+                    match o with
+                    | ExecutionOutput e -> Assert.AreEqual(e.ExitCode, expectedExitCode, sprintf "Exit code was expected to be: %A, but got %A." expectedExitCode e.ExitCode)
+                    | _ -> failwith "Cannot check exit code on this run result."
             result
 
         let private checkOutput (category: string) (substring: string) (selector: ExecutionOutput -> string) (result: TestResult) : TestResult =
@@ -636,9 +704,12 @@ module rec Compiler =
                 match r.Output with
                 | None -> failwith (sprintf "Execution output is missing cannot check \"%A\"" category)
                 | Some o ->
-                    let where = selector o
-                    if not (where.Contains(substring)) then
-                        failwith (sprintf "\nThe following substring:\n    %A\nwas not found in the %A\nOutput:\n    %A" substring category where)
+                    match o with
+                    | ExecutionOutput e ->
+                        let where = selector e
+                        if not (where.Contains(substring)) then
+                            failwith (sprintf "\nThe following substring:\n    %A\nwas not found in the %A\nOutput:\n    %A" substring category where)
+                    | _ -> failwith "Cannot check output on this run result."
             result
 
         let withOutputContains (substring: string) (result: TestResult) : TestResult =
@@ -649,3 +720,31 @@ module rec Compiler =
 
         let withStdErrContains (substring: string) (result: TestResult) : TestResult =
             checkOutput "STDERR" substring (fun o -> o.StdErr) result
+
+        // TODO: probably needs a bit of simplification, + need to remove that pyramid of doom.
+        let private assertEvalOutput (selector: FsiValue -> 'T) (value: 'T) (result: TestResult) : TestResult =
+            match result with
+            | Success r | Failure r ->
+                match r.Output with
+                | None -> failwith "Execution output is missing cannot check value."
+                | Some o ->
+                    match o with
+                    | EvalOutput e ->
+                        match e with
+                        | Ok v ->
+                            match v with
+                            | None -> failwith "Cannot assert value of evaluation, since it is None."
+                            | Some e -> Assert.AreEqual(value, (selector e))
+                        | Result.Error ex -> raise ex
+                    | _ -> failwith "Only 'eval' output is supported."
+            result
+
+        // TODO: Need to support for:
+        // STDIN, to test completions
+        // Contains
+        // Cancellation
+        let withEvalValueEquals (value: 'T) (result: TestResult) : TestResult =
+            assertEvalOutput (fun (x: FsiValue) -> x.ReflectionValue :?> 'T) value result
+
+        let withEvalTypeEquals t (result: TestResult) : TestResult =
+            assertEvalOutput (fun (x: FsiValue) -> x.ReflectionType) t result

tests/FSharp.Test.Utilities/CompilerAssert.fs

@@ -665,35 +665,39 @@ let main argv = 0"""
     static member CompileLibraryAndVerifyIL (source: string) (f: ILVerifier -> unit) =
         CompilerAssert.CompileLibraryAndVerifyILWithOptions [||] source f
 
-    static member RunScriptWithOptions options (source: string) (expectedErrorMessages: string list) =
-        lock gate <| fun () ->
-            // Intialize output and input streams
-            use inStream = new StringReader("")
-            use outStream = new StringWriter()
-            use errStream = new StringWriter()
-
-            // Build command line arguments & start FSI session
-            let argv = [| "C:\\fsi.exe" |]
-    #if NETCOREAPP
-            let args = Array.append argv [|"--noninteractive"; "--targetprofile:netcore"|]
-    #else
-            let args = Array.append argv [|"--noninteractive"; "--targetprofile:mscorlib"|]
-    #endif
-            let allArgs = Array.append args options
-
-            let fsiConfig = FsiEvaluationSession.GetDefaultConfiguration()
-            use fsiSession = FsiEvaluationSession.Create(fsiConfig, allArgs, inStream, outStream, errStream, collectible = true)
-
-            let ch, errors = fsiSession.EvalInteractionNonThrowing source
-
-            let errorMessages = ResizeArray()
-            errors
-            |> Seq.iter (fun error -> errorMessages.Add(error.Message))
+    static member RunScriptWithOptionsAndReturnResult options (source: string) =
+        // Intialize output and input streams
+        use inStream = new StringReader("")
+        use outStream = new StringWriter()
+        use errStream = new StringWriter()
+
+        // Build command line arguments & start FSI session
+        let argv = [| "C:\\fsi.exe" |]
+#if NETCOREAPP
+        let args = Array.append argv [|"--noninteractive"; "--targetprofile:netcore"|]
+#else
+        let args = Array.append argv [|"--noninteractive"; "--targetprofile:mscorlib"|]
+#endif
+        let allArgs = Array.append args options
 
-            match ch with
-            | Choice2Of2 ex -> errorMessages.Add(ex.Message)
-            | _ -> ()
+        let fsiConfig = FsiEvaluationSession.GetDefaultConfiguration()
+        use fsiSession = FsiEvaluationSession.Create(fsiConfig, allArgs, inStream, outStream, errStream, collectible = true)
 
+        let ch, errors = fsiSession.EvalInteractionNonThrowing source
+
+        let errorMessages = ResizeArray()
+        errors
+        |> Seq.iter (fun error -> errorMessages.Add(error.Message))
+
+        match ch with
+        | Choice2Of2 ex -> errorMessages.Add(ex.Message)
+        | _ -> ()
+
+        errorMessages
+
+    static member RunScriptWithOptions options (source: string) (expectedErrorMessages: string list) =
+        lock gate <| fun () ->
+            let errorMessages = CompilerAssert.RunScriptWithOptionsAndReturnResult options source
             if expectedErrorMessages.Length <> errorMessages.Count then
                 Assert.Fail(sprintf "Expected error messages: %A \n\n Actual error messages: %A" expectedErrorMessages errorMessages)
             else

tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj

@@ -35,6 +35,7 @@
     <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Core\FSharp.Core.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Build\FSharp.Build.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Compiler.Private\FSharp.Compiler.Private.fsproj" />
+    <ProjectReference Include="$(FSharpSourcesRoot)\fsharp\FSharp.Compiler.Private.Scripting\FSharp.Compiler.Private.Scripting.fsproj" />
     <ProjectReference Include="$(FSharpTestsRoot)\fsharpqa\testenv\src\PEVerify\PEVerify.csproj" />
   </ItemGroup>
 
@@ -56,6 +57,7 @@
         <PackageReference Include="Microsoft.CodeAnalysis.Test.Resources.Proprietary" Version="$(MicrosoftCodeAnalysisTestResourcesProprietaryVersion)" />
         <PackageReference Include="Microsoft.NETCore.App.Ref" Version="3.1.0" IncludeAssets="none" PrivateAssets="all" GeneratePathProperty="true" />
   </ItemGroup>
+
   <ItemGroup>
         <EmbeddedResource Include="$(PkgMicrosoft_NETCore_App_Ref)\ref\netcoreapp3.1\mscorlib.dll" LogicalName="netcoreapp31.mscorlib.dll">
                 <CopyToOutputDirectory>Always</CopyToOutputDirectory>

tests/FSharp.Test.Utilities/ILChecker.fs

@@ -134,7 +134,6 @@ module ILChecker =
 
     let verifyIL (dllFilePath: string) (expectedIL: string) =
         checkIL dllFilePath [expectedIL]
-
     let verifyILAndReturnActual (dllFilePath: string) (expectedIL: string) = checkILAux' [] dllFilePath [expectedIL]
     let reassembleIL ilFilePath dllFilePath =
         let ilasmPath = config.ILASM