Fixed parametrized tests

Damtev · Damtev · commit f8efdb468e5b · 2022-10-27T16:42:55.000+08:00
diff --git a/utbot-core/src/main/kotlin/org/utbot/common/HackUtil.kt b/utbot-core/src/main/kotlin/org/utbot/common/HackUtil.kt
@@ -67,4 +67,13 @@ enum class WorkaroundReason {
      * requires thorough [investigation](https://github.com/UnitTestBot/UTBotJava/issues/716).
      */
     IGNORE_STATICS_FROM_TRUSTED_LIBRARIES,
+    /**
+     * Methods that return [java.util.stream.BaseStream] as a result, can return them ”dirty” - consuming of them lead to the exception.
+     * The symbolic engine and concrete execution create UtStreamConsumingFailure executions in such cases. To warn a
+     * user about unsafety of using such “dirty” streams, code generation consumes them (mostly with `toArray` methods)
+     * and asserts exception. Unfortunately, it doesn't work well for parametrized tests - they create assertions relying on
+     * such-called “generic execution”, so resulted tests always contain `deepEquals` for streams, and we cannot easily
+     * construct `toArray` invocation (because streams cannot be consumed twice).
+     */
+    CONSUME_DIRTY_STREAMS,
 }
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/UtExecutionResult.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/UtExecutionResult.kt
@@ -11,6 +11,13 @@ data class UtExecutionSuccess(val model: UtModel) : UtExecutionResult() {
 
 sealed class UtExecutionFailure : UtExecutionResult() {
     abstract val exception: Throwable
+
+    /**
+     * Represents the most inner exception in the failure.
+     * Often equals to [exception], but is wrapped exception in [UtStreamConsumingException].
+     */
+    open val rootCauseException: Throwable
+        get() = exception
 }
 
 data class UtOverflowFailure(
@@ -22,8 +29,11 @@ data class UtSandboxFailure(
 ) : UtExecutionFailure()
 
 data class UtStreamConsumingFailure(
-    override val exception: Throwable
-) : UtExecutionFailure()
+    override val exception: UtStreamConsumingException,
+) : UtExecutionFailure() {
+    override val rootCauseException: Throwable
+        get() = exception.innerExceptionOrAny
+}
 
 /**
  * unexpectedFail (when exceptions such as NPE, IOBE, etc. appear, but not thrown by a user, applies both for function under test and nested calls )
@@ -46,8 +56,8 @@ class TimeoutException(s: String) : Exception(s)
 data class UtTimeoutException(override val exception: TimeoutException) : UtExecutionFailure()
 
 /**
- * Represents an exception that occurs during consuming a stream.
- * [innerException] stores original exception (if possible), null if [UtStreamConsumingException] was constructed by the engine.
+ * An artificial exception that stores an exception that would be thrown in case of consuming stream by invoking terminal operations.
+ * [innerException] stores this possible exception (null if [UtStreamConsumingException] was constructed by the engine).
  */
 data class UtStreamConsumingException(private val innerException: Exception?) : RuntimeException() {
     /**
@@ -101,7 +111,7 @@ inline fun UtExecutionResult.onSuccess(action: (model: UtModel) -> Unit): UtExec
 }
 
 inline fun UtExecutionResult.onFailure(action: (exception: Throwable) -> Unit): UtExecutionResult {
-    if (this is UtExecutionFailure) action(exception)
+    if (this is UtExecutionFailure) action(rootCauseException)
     return this
 }
 
@@ -111,6 +121,6 @@ fun UtExecutionResult.getOrThrow(): UtModel = when (this) {
 }
 
 fun UtExecutionResult.exceptionOrNull(): Throwable? = when (this) {
-    is UtExecutionFailure -> exception
+    is UtExecutionFailure -> rootCauseException
     is UtExecutionSuccess -> null
 }
diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/examples/stream/StreamsAsMethodResultExampleTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/examples/stream/StreamsAsMethodResultExampleTest.kt
@@ -20,7 +20,6 @@ class StreamsAsMethodResultExampleTest : UtValueTestCaseChecker(
         TestLastStage(CodegenLanguage.JAVA),
         TestLastStage(CodegenLanguage.KOTLIN, CodeGeneration)
     ),
-    codeGenerationModes = listOf(ParametrizedTestSource.DO_NOT_PARAMETRIZE) // TODO exception from concrete is passed to arguments list somehow
 ) {
     @Test
     fun testReturningStreamExample() {
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt
@@ -90,6 +90,8 @@ import kotlin.math.max
 import kotlin.math.min
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.persistentSetOf
+import org.utbot.framework.plugin.api.UtStreamConsumingException
+import org.utbot.framework.plugin.api.UtStreamConsumingFailure
 
 // hack
 const val MAX_LIST_SIZE = 10
@@ -370,6 +372,12 @@ class Resolver(
      */
     private fun SymbolicFailure.resolve(): UtExecutionFailure {
         val exception = concreteException()
+
+        if (exception is UtStreamConsumingException) {
+            // This exception is artificial and is not really thrown
+            return UtStreamConsumingFailure(exception)
+        }
+
         return if (explicit) {
             UtExplicitlyThrownException(exception, inNestedMethod)
         } else {
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt
@@ -1,7 +1,9 @@
 package org.utbot.framework.codegen.model.constructor.tree
 
 import org.utbot.common.PathUtil
+import org.utbot.common.WorkaroundReason
 import org.utbot.common.isStatic
+import org.utbot.common.workaround
 import org.utbot.framework.assemble.assemble
 import org.utbot.framework.codegen.ForceStaticMocking
 import org.utbot.framework.codegen.ParametrizedTestSource
@@ -146,7 +148,8 @@ import java.lang.reflect.InvocationTargetException
 import java.security.AccessControlException
 import java.lang.reflect.ParameterizedType
 import org.utbot.framework.UtSettings
-import org.utbot.framework.plugin.api.UtStreamConsumingException
+import org.utbot.framework.plugin.api.UtExecutionResult
+import org.utbot.framework.plugin.api.UtStreamConsumingFailure
 import org.utbot.framework.plugin.api.util.allSuperTypes
 import org.utbot.framework.plugin.api.util.baseStreamClassId
 import org.utbot.framework.plugin.api.util.doubleStreamClassId
@@ -183,6 +186,12 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
 
     private val fieldsOfExecutionResults = mutableMapOf<Pair<FieldId, Int>, MutableList<UtModel>>()
 
+    /**
+     * Contains whether [UtStreamConsumingFailure] is in [CgMethodTestSet] for parametrized tests.
+     * See [WorkaroundReason.CONSUME_DIRTY_STREAMS].
+     */
+    private var containsStreamConsumingFailureForParametrizedTests: Boolean = false
+
     private fun setupInstrumentation() {
         if (currentExecution is UtSymbolicExecution) {
             val execution = currentExecution as UtSymbolicExecution
@@ -300,28 +309,30 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                 emptyLineIfNeeded()
                 val method = currentExecutable as MethodId
                 val currentExecution = currentExecution!!
+                val executionResult = currentExecution.result
+
                 // build assertions
-                currentExecution.result
-                    .onSuccess { result ->
+                executionResult
+                    .onSuccess { resultModel ->
                         methodType = SUCCESSFUL
 
-                        // TODO possible engine bug - void method return type and result not UtVoidModel
-                        if (result.isUnit() || method.returnType == voidClassId) {
+                        // TODO possible engine bug - void method return type and result model not UtVoidModel
+                        if (resultModel.isUnit() || method.returnType == voidClassId) {
                             +thisInstance[method](*methodArguments.toTypedArray())
                         } else {
-                            resultModel = result
-                            val expected = variableConstructor.getOrCreateVariable(result, "expected")
+                            this.resultModel = resultModel
+                            val expected = variableConstructor.getOrCreateVariable(resultModel, "expected")
                             assertEquality(expected, actual)
                         }
                     }
-                    .onFailure { exception -> processExecutionFailure(exception) }
+                    .onFailure { exception -> processExecutionFailure(exception, executionResult) }
             }
             else -> {} // TODO: check this specific case
         }
     }
 
-    private fun processExecutionFailure(exceptionFromAnalysis: Throwable) {
-        val (methodInvocationBlock, expectedException) = constructExceptionProducingBlock(exceptionFromAnalysis)
+    private fun processExecutionFailure(exceptionFromAnalysis: Throwable, executionResult: UtExecutionResult) {
+        val (methodInvocationBlock, expectedException) = constructExceptionProducingBlock(exceptionFromAnalysis, executionResult)
 
         when (methodType) {
             SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $expectedException")
@@ -355,9 +366,12 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
         }
     }
 
-    private fun constructExceptionProducingBlock(exceptionFromAnalysis: Throwable): Pair<() -> Unit, Throwable> {
-        if (exceptionFromAnalysis is UtStreamConsumingException) {
-            return constructStreamConsumingBlock() to exceptionFromAnalysis.innerExceptionOrAny
+    private fun constructExceptionProducingBlock(
+        exceptionFromAnalysis: Throwable,
+        executionResult: UtExecutionResult
+    ): Pair<() -> Unit, Throwable> {
+        if (executionResult is UtStreamConsumingFailure) {
+            return constructStreamConsumingBlock() to executionResult.rootCauseException
         }
 
         return {
@@ -461,22 +475,32 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
             is ConstructorId -> generateConstructorCall(currentExecutable!!, currentExecution!!)
             is MethodId -> {
                 val method = currentExecutable as MethodId
-                currentExecution!!.result
-                    .onSuccess { result ->
-                        if (result.isUnit()) {
+                val executionResult = currentExecution!!.result
+
+                executionResult
+                    .onSuccess { resultModel ->
+                        if (resultModel.isUnit()) {
                             +thisInstance[method](*methodArguments.toTypedArray())
                         } else {
                             //"generic" expected variable is represented with a wrapper if
                             //actual result is primitive to support cases with exceptions.
-                            resultModel = if (result is UtPrimitiveModel) assemble(result) else result
+                            this.resultModel = if (resultModel is UtPrimitiveModel) assemble(resultModel) else resultModel
 
                             val expectedVariable = currentMethodParameters[CgParameterKind.ExpectedResult]!!
                             val expectedExpression = CgNotNullAssertion(expectedVariable)
 
                             assertEquality(expectedExpression, actual)
                         }
                     }
-                    .onFailure { thisInstance[method](*methodArguments.toTypedArray()).intercepted() }
+                    .onFailure {
+                        workaround(WorkaroundReason.CONSUME_DIRTY_STREAMS) {
+                            if (containsStreamConsumingFailureForParametrizedTests) {
+                                constructStreamConsumingBlock().invoke()
+                            } else {
+                                thisInstance[method](*methodArguments.toTypedArray()).intercepted()
+                            }
+                        }
+                    }
             }
             else -> {} // TODO: check this specific case
         }
@@ -1208,7 +1232,9 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
         // we cannot generate any assertions for constructor testing
         // but we need to generate a constructor call
         val constructorCall = currentExecutableId as ConstructorId
-        currentExecution.result
+        val executionResult = currentExecution.result
+
+        executionResult
             .onSuccess {
                 methodType = SUCCESSFUL
 
@@ -1220,7 +1246,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                     constructorCall(*methodArguments.toTypedArray())
                 }
             }
-            .onFailure { exception -> processExecutionFailure(exception) }
+            .onFailure { exception -> processExecutionFailure(exception, executionResult) }
     }
 
     /**
@@ -1241,7 +1267,15 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                     actual.type.isPrimitive -> generateDeepEqualsAssertion(expected, actual)
                     else -> ifStatement(
                         CgEqualTo(expected, nullLiteral()),
-                        trueBranch = { +testFrameworkManager.assertions[testFramework.assertNull](actual).toStatement() },
+                        trueBranch = {
+                            workaround(WorkaroundReason.CONSUME_DIRTY_STREAMS) {
+                                if (containsStreamConsumingFailureForParametrizedTests) {
+                                    constructStreamConsumingBlock().invoke()
+                                } else {
+                                    +testFrameworkManager.assertions[testFramework.assertNull](actual).toStatement()
+                                }
+                            }
+                        },
                         falseBranch = {
                             +testFrameworkManager.assertions[testFrameworkManager.assertNotNull](actual).toStatement()
                             generateDeepEqualsAssertion(expected, actual)
@@ -1270,21 +1304,23 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
     }
 
     private fun recordActualResult() {
-        currentExecution!!.result.onSuccess { result ->
+        val executionResult = currentExecution!!.result
+
+        executionResult.onSuccess { resultModel ->
             when (val executable = currentExecutable) {
                 is ConstructorId -> {
                     // there is nothing to generate for constructors
                     return
                 }
                 is BuiltinMethodId -> error("Unexpected BuiltinMethodId $currentExecutable while generating actual result")
                 is MethodId -> {
-                    // TODO possible engine bug - void method return type and result not UtVoidModel
-                    if (result.isUnit() || executable.returnType == voidClassId) return
+                    // TODO possible engine bug - void method return type and result model not UtVoidModel
+                    if (resultModel.isUnit() || executable.returnType == voidClassId) return
 
                     emptyLineIfNeeded()
 
                     actual = newVar(
-                        CgClassId(result.classId, isNullable = result is UtNullModel),
+                        CgClassId(resultModel.classId, isNullable = resultModel is UtNullModel),
                         "actual"
                     ) {
                         thisInstance[executable](*methodArguments.toTypedArray())
@@ -1293,13 +1329,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                 else -> {} // TODO: check this specific case
             }
         }.onFailure {
-            processActualInvocationFailure(it)
+            processActualInvocationFailure(executionResult)
         }
     }
 
-    private fun processActualInvocationFailure(e: Throwable) {
-        when (e) {
-            is UtStreamConsumingException -> processStreamConsumingException(e.innerExceptionOrAny)
+    private fun processActualInvocationFailure(executionResult: UtExecutionResult) {
+        when (executionResult) {
+            is UtStreamConsumingFailure -> processStreamConsumingException(executionResult.rootCauseException)
             else -> {} // Do nothing for now
         }
     }
@@ -1423,6 +1459,12 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
         //may be a heuristic to select a model with minimal number of internal nulls should be used
         val genericExecution = chooseGenericExecution(testSet.executions)
 
+        workaround(WorkaroundReason.CONSUME_DIRTY_STREAMS) {
+            containsStreamConsumingFailureForParametrizedTests = testSet.executions.any {
+                it.result is UtStreamConsumingFailure
+            }
+        }
+
         val statics = genericExecution.stateBefore.statics
 
         return withTestMethodScope(genericExecution) {
diff --git a/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/CodeGenerationIntegrationTest.kt b/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/CodeGenerationIntegrationTest.kt
@@ -33,8 +33,7 @@ abstract class CodeGenerationIntegrationTest(
     private val languagesLastStages: List<TestLastStage> = listOf(
         TestLastStage(CodegenLanguage.JAVA),
         TestLastStage(CodegenLanguage.KOTLIN)
-    ),
-    private val codeGenerationModes: List<ParametrizedTestSource> = parameterizationModes
+    )
 ) {
     private val testSets: MutableList<UtMethodTestSet> = arrayListOf()
 
@@ -100,7 +99,7 @@ abstract class CodeGenerationIntegrationTest(
 
             // TODO: leave kotlin & parameterized mode configuration alone for now
             val pipelineConfigurations = languages
-                .flatMap { language -> codeGenerationModes.map { mode -> language to mode } }
+                .flatMap { language -> parameterizationModes.map { mode -> language to mode } }
                 .filterNot { it == CodegenLanguage.KOTLIN to ParametrizedTestSource.PARAMETRIZE }
 
             for ((language, parameterizationMode) in pipelineConfigurations) {
@@ -163,7 +162,7 @@ abstract class CodeGenerationIntegrationTest(
 
         private val languages = listOf(CodegenLanguage.JAVA, CodegenLanguage.KOTLIN)
 
-        internal val parameterizationModes = listOf(ParametrizedTestSource.DO_NOT_PARAMETRIZE, ParametrizedTestSource.PARAMETRIZE)
+        private val parameterizationModes = listOf(ParametrizedTestSource.DO_NOT_PARAMETRIZE, ParametrizedTestSource.PARAMETRIZE)
 
         data class CodeGenerationTestCases(
             val testClass: KClass<*>,
diff --git a/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/UtValueTestCaseChecker.kt b/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/UtValueTestCaseChecker.kt
@@ -14,7 +14,6 @@ import org.utbot.framework.UtSettings.daysLimitForTempFiles
 import org.utbot.framework.UtSettings.testDisplayName
 import org.utbot.framework.UtSettings.testName
 import org.utbot.framework.UtSettings.testSummary
-import org.utbot.framework.codegen.ParametrizedTestSource
 import org.utbot.framework.coverage.Coverage
 import org.utbot.framework.coverage.counters
 import org.utbot.framework.coverage.methodCoverage
@@ -73,9 +72,8 @@ abstract class UtValueTestCaseChecker(
     pipelines: List<TestLastStage> = listOf(
         TestLastStage(CodegenLanguage.JAVA),
         TestLastStage(CodegenLanguage.KOTLIN)
-    ),
-    codeGenerationModes: List<ParametrizedTestSource> = parameterizationModes
-) : CodeGenerationIntegrationTest(testClass, testCodeGeneration, pipelines, codeGenerationModes) {
+    )
+) : CodeGenerationIntegrationTest(testClass, testCodeGeneration, pipelines) {
     // contains already analyzed by the engine methods
     private val analyzedMethods: MutableMap<MethodWithMockStrategy, MethodResult> = mutableMapOf()
 
