From 9f58993f9630c8250a9558c60e5ece817c2ccb52 Mon Sep 17 00:00:00 2001
From: BooksBaum <15612932+Booksbaum@users.noreply.github.com>
Date: Fri, 1 Jul 2022 17:13:18 +0200
Subject: [PATCH] Fix CI Tests (#955)

---
 .github/workflows/build.yml                   |   2 +-
 .../RenameParamToMatchSignatureTests.fs       | 124 +++++++++---------
 .../CodeFixTests/Tests.fs                     |  22 ++--
 .../DependentFileCheckingTests.fs             |  90 +++++++------
 test/FsAutoComplete.Tests.Lsp/Utils/Server.fs |  21 ++-
 5 files changed, 147 insertions(+), 112 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3705090a7..4408de896 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -9,7 +9,7 @@ jobs:
     timeout-minutes: 20 # we have a locking issue, so cap the runs at ~20m to account for varying build times, etc
     strategy:
       matrix:
-        os: [windows-2019, macos-10.15, ubuntu-20.04]
+        os: [windows-2022, macos-11, ubuntu-20.04]
         dotnet: [6.0.200]
       fail-fast: false # we have timing issues on some OS, so we want them all to run
 
diff --git a/test/FsAutoComplete.Tests.Lsp/CodeFixTests/RenameParamToMatchSignatureTests.fs b/test/FsAutoComplete.Tests.Lsp/CodeFixTests/RenameParamToMatchSignatureTests.fs
index 5973fcf82..daa72bc34 100644
--- a/test/FsAutoComplete.Tests.Lsp/CodeFixTests/RenameParamToMatchSignatureTests.fs
+++ b/test/FsAutoComplete.Tests.Lsp/CodeFixTests/RenameParamToMatchSignatureTests.fs
@@ -53,78 +53,78 @@ let tests state =
         """
         module Code
 
-        val f: value: int -> int
+        val f: value1: int -> int
         """
         """
         module Code
 
-        let f $0v = v + 1
+        let f $0v1 = v1 + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value1")
         """
         module Code
 
-        let f value = value + 1
+        let f value1 = value1 + 1
         """
     testCaseAsync "can rename parameter with backticks in signature in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: ``my value``: int -> int
+        val f: ``my value2``: int -> int
         """
         """
         module Code
 
-        let f $0v = v + 1
+        let f $0v2 = v2 + 1
         """
-        (selectCodeFix "``my value``")
+        (selectCodeFix "``my value2``")
         """
         module Code
 
-        let f ``my value`` = ``my value`` + 1
+        let f ``my value2`` = ``my value2`` + 1
         """
     testCaseAsync "can rename parameter with backticks in implementation in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value: int -> int
+        val f: value3: int -> int
         """
         """
         module Code
 
-        let f ``$0my value`` = ``my value`` + 1
+        let f ``$0my value3`` = ``my value3`` + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value3")
         """
         module Code
 
-        let f value = value + 1
+        let f value3 = value3 + 1
         """
     testCaseAsync "can rename all usage in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: x: int -> value: int -> y: int -> int
+        val f: x: int -> value4: int -> y: int -> int
         """
         """
         module Code
 
-        let f x $0v y =
-          let a = v + 1
-          let b = v * v
+        let f x $0v4 y =
+          let a = v4 + 1
+          let b = v4 * v4
           let v = a + b
           v + x * y
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value4")
         """
         module Code
 
-        let f x value y =
-          let a = value + 1
-          let b = value * value
+        let f x value4 y =
+          let a = value4 + 1
+          let b = value4 * value4
           let v = a + b
           v + x * y
         """
@@ -133,18 +133,18 @@ let tests state =
         """
         module Code
 
-        val f: value: int -> int
+        val f: value5: int -> int
         """
         """
         module Code
 
-        let f ($0v: int) = v + 1
+        let f ($0v5: int) = v5 + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value5")
         """
         module Code
 
-        let f (value: int) = value + 1
+        let f (value5: int) = value5 + 1
         """
     testCaseAsync "can rename parameter in constructor" <|
       checkWithFsi
@@ -152,20 +152,20 @@ let tests state =
         module Code
 
         type T =
-          new: value: int -> T
+          new: value6: int -> T
         """
         """
         module Code
 
-        type T($0v: int) =
-          let _ = v + 3
+        type T($0v6: int) =
+          let _ = v6 + 3
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value6")
         """
         module Code
 
-        type T(value: int) =
-          let _ = value + 3
+        type T(value6: int) =
+          let _ = value6 + 3
         """
     testCaseAsync "can rename parameter in member" <|
       checkWithFsi
@@ -174,146 +174,146 @@ let tests state =
 
         type T =
           new: unit -> T
-          member F: value: int -> int
+          member F: value7: int -> int
         """
         """
         module Code
 
         type T() =
-          member _.F($0v) = v + 1
+          member _.F($0v7) = v7 + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value7")
         """
         module Code
 
         type T() =
-          member _.F(value) = value + 1
+          member _.F(value7) = value7 + 1
         """
     testCaseAsync "can rename parameter with ' in signature in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value': int -> int
+        val f: value8': int -> int
         """
         """
         module Code
 
-        let f $0v = v + 1
+        let f $0v8 = v8 + 1
         """
-        (selectCodeFix "value'")
+        (selectCodeFix "value8'")
         """
         module Code
 
-        let f value' = value' + 1
+        let f value8' = value8' + 1
         """
     testCaseAsync "can rename parameter with ' in implementation in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value: int -> int
+        val f: value9: int -> int
         """
         """
         module Code
 
-        let f $0v' = v' + 1
+        let f $0v9' = v9' + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value9")
         """
         module Code
 
-        let f value = value + 1
+        let f value9 = value9 + 1
         """
     testCaseAsync "can rename parameter with ' (not in last place) in signature in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: v'2: int -> int
+        val f: v10'2: int -> int
         """
         """
         module Code
 
-        let f $0value = value + 1
+        let f $0value10 = value10 + 1
         """
-        (selectCodeFix "v'2")
+        (selectCodeFix "v10'2")
         """
         module Code
 
-        let f v'2 = v'2 + 1
+        let f v10'2 = v10'2 + 1
         """
     testCaseAsync "can rename parameter with ' (not in last place) in implementation in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value: int -> int
+        val f: value11: int -> int
         """
         """
         module Code
 
-        let f $0v'2 = v'2 + 1
+        let f $0v11'2 = v11'2 + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value11")
         """
         module Code
 
-        let f value = value + 1
+        let f value11 = value11 + 1
         """
     testCaseAsync "can rename parameter with multiple ' in signature in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value'v'2: int -> int
+        val f: value12'v'2: int -> int
         """
         """
         module Code
 
-        let f $0v = v + 1
+        let f $0v12 = v12 + 1
         """
-        (selectCodeFix "value'v'2")
+        (selectCodeFix "value12'v'2")
         """
         module Code
 
-        let f value'v'2 = value'v'2 + 1
+        let f value12'v'2 = value12'v'2 + 1
         """
     testCaseAsync "can rename parameter with multiple ' in implementation in F# function" <|
       checkWithFsi
         """
         module Code
 
-        val f: value: int -> int
+        val f: value13: int -> int
         """
         """
         module Code
 
-        let f $0value'v'2 = value'v'2 + 1
+        let f $0value13'v'2 = value13'v'2 + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value13")
         """
         module Code
 
-        let f value = value + 1
+        let f value13 = value13 + 1
         """
     itestCaseAsync "can handle `' and implementation '` in impl name" <|
       checkWithFsi
         """
         module Code
 
-        val f: value: int -> int
+        val f: value14: int -> int
         """
         """
         module Code
 
         let f $0``sig' and implementation 'impl' do not match`` = ``sig' and implementation 'impl' do not match`` + 1
         """
-        (selectCodeFix "value")
+        (selectCodeFix "value14")
         """
         module Code
 
-        let f value = value + 1
+        let f value14 = value14 + 1
         """
     //ENHANCEMENT: correctly detect below. Currently: detects sig name `sig`
     itestCaseAsync "can handle `' and implementation '` in sig name" <|
@@ -326,7 +326,7 @@ let tests state =
         """
         module Code
 
-        let f $0value = value + 1
+        let f $0value15 = value15 + 1
         """
         (selectCodeFix "``sig' and implementation 'impl' do not match``")
         """
diff --git a/test/FsAutoComplete.Tests.Lsp/CodeFixTests/Tests.fs b/test/FsAutoComplete.Tests.Lsp/CodeFixTests/Tests.fs
index adffad2a2..8a0defc8c 100644
--- a/test/FsAutoComplete.Tests.Lsp/CodeFixTests/Tests.fs
+++ b/test/FsAutoComplete.Tests.Lsp/CodeFixTests/Tests.fs
@@ -1273,12 +1273,18 @@ let private renameUnusedValue state =
 let private replaceWithSuggestionTests state =
   serverTestList (nameof ReplaceWithSuggestion) state defaultConfigDto None (fun server -> [
     let selectCodeFix replacement = CodeFix.withTitle (ReplaceWithSuggestion.title replacement)
+    let validateDiags (diags: Diagnostic[]) =
+      Diagnostics.expectCode "39" diags
+      Expect.exists 
+        diags 
+        (fun (d: Diagnostic) -> d.Message.Contains "Maybe you want one of the following:")
+        "Diagnostic with code 39 should suggest name"
     testCaseAsync "can change Min to min" <|
       CodeFix.check server
         """
         let x = $0Min(2.0, 1.0)
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "min")
         """
         let x = min(2.0, 1.0)
@@ -1289,7 +1295,7 @@ let private replaceWithSuggestionTests state =
           """
           let x = $0flout 2
           """
-          Diagnostics.acceptAll
+          validateDiags
           (selectCodeFix "float")
           """
           let x = float 2
@@ -1299,7 +1305,7 @@ let private replaceWithSuggestionTests state =
           """
           let x = $0flout 2
           """
-          Diagnostics.acceptAll
+          validateDiags
           (selectCodeFix "float32")
           """
           let x = float32 2
@@ -1310,7 +1316,7 @@ let private replaceWithSuggestionTests state =
         """
         let x: $0flout = 2.0
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "float")
         """
         let x: float = 2.0
@@ -1320,7 +1326,7 @@ let private replaceWithSuggestionTests state =
         """
         open System.Text.$0RegularEcpressions
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "RegularExpressions")
         """
         open System.Text.RegularExpressions
@@ -1331,7 +1337,7 @@ let private replaceWithSuggestionTests state =
         open System.Text.RegularExpressions
         let x = $0Regec()
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "Regex")
         """
         open System.Text.RegularExpressions
@@ -1343,7 +1349,7 @@ let private replaceWithSuggestionTests state =
         let ``hello world`` = 2
         let x = ``$0hello word``
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "``hello world``")
         """
         let ``hello world`` = 2
@@ -1355,7 +1361,7 @@ let private replaceWithSuggestionTests state =
         let ``hello world`` = 2
         let x = $0helloword
         """
-        Diagnostics.acceptAll
+        validateDiags
         (selectCodeFix "``hello world``")
         """
         let ``hello world`` = 2
diff --git a/test/FsAutoComplete.Tests.Lsp/DependentFileCheckingTests.fs b/test/FsAutoComplete.Tests.Lsp/DependentFileCheckingTests.fs
index 55b49389c..b317fc9e8 100644
--- a/test/FsAutoComplete.Tests.Lsp/DependentFileCheckingTests.fs
+++ b/test/FsAutoComplete.Tests.Lsp/DependentFileCheckingTests.fs
@@ -6,56 +6,66 @@ open Helpers
 open System.IO
 open Utils.Tests
 open Utils.Server
-open Utils.Server.Document
 open System
 open FSharp.Control.Reactive
-open System.Reactive.Linq
 open FSharpx.Control
 
 let tests state =
   let root = Path.Combine(__SOURCE_DIRECTORY__, "TestCases", "DependentFileChecking", "SameProject")
   let aFile, bFile = "A.fs", "B.fs"
-  testSequenced <| serverTestList (nameof DependentFileChecking) state defaultConfigDto (Some root) (fun server -> [
-    testCaseAsync "When A is modified B is re-checked" (async {
-      // open the files as they are on-disk and verify things are good
-      let! (aDoc, aDiags) = Server.openDocument aFile server
-      let! (bDoc, bDiags) = Server.openDocument bFile server
-      use aDoc = aDoc
-      use bDoc = bDoc
-      Expect.isEmpty aDiags $"There should be no diagnostics in {aFile}"
-      Expect.isEmpty bDiags $"There should be no diagnostics in {bFile}"
-
-      let bDiagsStream = bDoc.CompilerDiagnostics
-      // make a change to A (that is clearly incorrect)
-      let! startAChange = Document.changeTextTo "farts" aDoc |> Async.StartChild
-      // start listening for new diagnostics for B
-      let! diagnosticsForBWaiter = bDiagsStream |> Observable.timeoutSpan (TimeSpan.FromSeconds 100.) |> Async.AwaitObservable |> Async.StartChild
-      let! aDiags = startAChange
-      Expect.isNonEmpty aDiags $"Should have had some compilation errors for {aFile} after erroneous changes"
-      // observe that compilation errors are reported for B
-      let! bDiags = diagnosticsForBWaiter
-      Expect.isNonEmpty bDiags $"Should have some compilation errors for {bFile} after erroneous change to {aFile}"
-    })
-
-    testCaseAsync "When A is modified repeatedly, B is re-checked after each modification" (async {
-      // open the files as they are on-disk and verify things are good
-      let! (aDoc, aDiags) = Server.openDocument aFile server
-      let! (bDoc, bDiags) = Server.openDocument bFile server
-      use aDoc = aDoc
-      use bDoc = bDoc
-      Expect.isEmpty aDiags $"There should be no diagnostics in {aFile}"
-      Expect.isEmpty bDiags $"There should be no diagnostics in {bFile}"
-      let bDiagsStream = bDoc.CompilerDiagnostics
-      for i in 0..10 do
+  testList (nameof DependentFileChecking) [
+    // Separate server for each test
+    // Otherwise share diag stream -> second test must skip diags of first tests. But only when first test runs (-> running all tests vs. running test alone)
+    serverTestList "single" state defaultConfigDto (Some root) (fun server -> [
+      testCaseAsync "When A is modified B is re-checked" (async {
+        // open the files as they are on-disk and verify things are good
+        let! (aDoc, aDiags) = Server.openDocument aFile server
+        let! (bDoc, bDiags) = Server.openDocument bFile server
+        use aDoc = aDoc
+        use bDoc = bDoc
+        Expect.isEmpty aDiags $"There should be no diagnostics in {aFile}"
+        Expect.isEmpty bDiags $"There should be no diagnostics in {bFile}"
+        let bDiagsStream = bDoc.CompilerDiagnostics
         // make a change to A (that is clearly incorrect)
         let! startAChange = Document.changeTextTo "farts" aDoc |> Async.StartChild
         // start listening for new diagnostics for B
-        let! diagnosticsForBWaiter = bDiagsStream |> Observable.skip (1 + i) |> Observable.timeoutSpan (TimeSpan.FromSeconds 100.) |> Async.AwaitObservable |> Async.StartChild
+        let! diagnosticsForBWaiter = 
+          bDiagsStream 
+          |> Observable.timeoutSpan (TimeSpan.FromSeconds 15.) 
+          |> Async.AwaitObservable 
+          |> Async.StartChild
         let! aDiags = startAChange
-        Expect.isNonEmpty aDiags $"Should have had some compilation errors for {aFile} after erroneous change #%d{i}"
+        Expect.isNonEmpty aDiags $"Should have had some compilation errors for {aFile} after erroneous changes"
         // observe that compilation errors are reported for B
         let! bDiags = diagnosticsForBWaiter
-        Expect.isNonEmpty bDiags $"Should have some compilation errors for {bFile} after erroneous change #%d{i} to {aFile}"
-    })
-
-  ])
+        Expect.isNonEmpty bDiags $"Should have some compilation errors for {bFile} after erroneous change to {aFile}"
+      })
+    ])
+    serverTestList "multi" state defaultConfigDto (Some root) (fun server -> [
+      testCaseAsync "When A is modified repeatedly, B is re-checked after each modification" (async {
+        // open the files as they are on-disk and verify things are good
+        let! (aDoc, aDiags) = Server.openDocument aFile server
+        let! (bDoc, bDiags) = Server.openDocument bFile server
+        use aDoc = aDoc
+        use bDoc = bDoc
+        Expect.isEmpty aDiags $"There should be no diagnostics in {aFile}"
+        Expect.isEmpty bDiags $"There should be no diagnostics in {bFile}"
+        let bDiagsStream = bDoc.CompilerDiagnostics
+        for i in 0..10 do
+          // make a change to A (that is clearly incorrect)
+          let! startAChange = Document.changeTextTo "farts" aDoc |> Async.StartChild
+          // start listening for new diagnostics for B
+          let! diagnosticsForBWaiter = 
+            bDiagsStream 
+            |> Observable.skip i
+            |> Observable.timeoutSpan (TimeSpan.FromSeconds 15.) 
+            |> Async.AwaitObservable 
+            |> Async.StartChild
+          let! aDiags = startAChange
+          Expect.isNonEmpty aDiags $"Should have had some compilation errors for {aFile} after erroneous change #%d{i}"
+          // observe that compilation errors are reported for B
+          let! bDiags = diagnosticsForBWaiter
+          Expect.isNonEmpty bDiags $"Should have some compilation errors for {bFile} after erroneous change #%d{i} to {aFile}"
+      })
+    ])
+  ]
diff --git a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs
index 7d16f2dfa..cd5bc14ca 100644
--- a/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs
+++ b/test/FsAutoComplete.Tests.Lsp/Utils/Server.fs
@@ -230,6 +230,25 @@ module Document =
     |> Observable.filter (fun n -> n.TextDocument.Uri = doc.Uri)
 
 
+  /// in ms
+  let private waitForLateDiagnosticsDelay =
+    let envVar = "FSAC_WaitForLateDiagnosticsDelay"
+    System.Environment.GetEnvironmentVariable envVar
+    |> Option.ofObj
+    |> Option.map (fun d ->
+      match System.Int32.TryParse d with
+      | (true, d) -> d
+      | (false, _) ->
+          failwith $"Environment Variable '%s{envVar}' exists, but is not a correct int number ('%s{d}')"
+    )
+    |> Option.orElseWith (fun _ -> 
+        // set in Github Actions: https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+        match System.Environment.GetEnvironmentVariable "CI" with
+        | null -> None
+        | _ -> Some 25
+    )
+    |> Option.defaultValue 7  // testing locally
+
   /// Waits (if necessary) and gets latest diagnostics.
   ///
   /// To detect newest diags:
@@ -281,7 +300,7 @@ module Document =
           |> analyzedStream
           |> Observable.filter (fun n -> n.TextDocument.Version = Some doc.Version)
           // wait for late diagnostics
-          |> Observable.delay 5
+          |> Observable.delay waitForLateDiagnosticsDelay
         )
         |> Observable.last
         |> Observable.timeoutSpan timeout