Add test for multirun and check gold value on the whole execution output

senagbe · Mar 27, 2019 · ce9f2b7 · ce9f2b7
1 parent 83b1db8
commit ce9f2b7
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 20 deletions.
diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle
@@ -2233,6 +2233,16 @@ task args0(type: RunStandaloneKonanTest) {
     source = "runtime/basic/args0.kt"
 }
 
+task multiargs(type: RunStandaloneKonanTest) {
+    arguments = ["AAA", "BB", "C"]
+    multiRuns = true
+    multiArguments = [["1", "2", "3"], ["---"], ["Hello", "world"]]
+    goldValue = "AAA\nBB\nC\n1\n2\n3" +
+            "AAA\nBB\nC\n---" +
+            "AAA\nBB\nC\nHello\nworld\n"
+    source = "runtime/basic/args0.kt"
+}
+
 task spread_operator_0(type: RunKonanTest) {
     goldValue = "[K, o, t, l, i, n,  , i, s,  , c, o, o, l,  , l, a, n, g, u, a, g, e]\n"
     source = "codegen/basics/spread_operator_0.kt"

diff --git a/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy
@@ -287,6 +287,8 @@ abstract class KonanTest extends JavaExec {
 
         def times = multiRuns ? multiArguments.size() : 1
 
+        def result = compilerMessagesText
+        def exitCodeMismatch = false
         for (int i = 0; i < times; i++) {
             ExecResult execResult = project.execute {
 
@@ -305,11 +307,11 @@ abstract class KonanTest extends JavaExec {
 
                 ignoreExitValue = true
             }
-            def result = compilerMessagesText + out.toString("UTF-8")
+            def currentResult = out.toString("UTF-8")
+            println(currentResult)
+            result += currentResult
 
-            println(result)
-
-            def exitCodeMismatch = execResult.exitValue != expectedExitStatus
+            exitCodeMismatch |= execResult.exitValue != expectedExitStatus
             if (exitCodeMismatch) {
                 def message = "Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}"
                 if (this.expectedFail) {
@@ -318,25 +320,25 @@ abstract class KonanTest extends JavaExec {
                     throw new TestFailedException("Test failed on iteration $i. $message")
                 }
             }
+        }
 
-            result = result.replace(System.lineSeparator(), "\n")
-            def goldValueMismatch = !outputChecker.apply(result)
-            if (goldValueMismatch) {
-                def message
-                if (goldValue != null) {
-                    message = "Iteration $i. Expected output: $goldValue, actual output: $result"
-                } else {
-                    message = "Iteration $i. Actual output doesn't match output checker: $result"
-                }
-                if (this.expectedFail) {
-                    println("Expected failure. $message")
-                } else {
-                    throw new TestFailedException("Test failed on iteration $i. $message")
-                }
+        result = result.replace(System.lineSeparator(), "\n")
+        def goldValueMismatch = !outputChecker.apply(result)
+        if (goldValueMismatch) {
+            def message
+            if (goldValue != null) {
+                message = "Expected output: $goldValue, actual output: $result"
+            } else {
+                message = "Actual output doesn't match output checker: $result"
+            }
+            if (this.expectedFail) {
+                println("Expected failure. $message")
+            } else {
+                throw new TestFailedException("Test failed. $message")
             }
-
-            if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
         }
+
+        if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
     }
 }