Field assertions fixes (UnitTestBot#920)

Damtev · AbdullinAM · commit a55999ddd72a · 2022-10-17T17:03:55.000+03:00
* Used real variable type after field access with reflection if possible

* Added missing line separator after single-line block comment

* Added missed initial field states for arrays

* Removed field state assertions for failing tests
diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateTypeTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateTypeTest.kt
@@ -0,0 +1,41 @@
+package org.utbot.examples.codegen.modifiers
+
+import org.junit.jupiter.api.Test
+import org.utbot.common.withAccessibility
+import org.utbot.framework.plugin.api.CodegenLanguage
+import org.utbot.framework.plugin.api.FieldId
+import org.utbot.framework.plugin.api.util.id
+import org.utbot.framework.plugin.api.util.jField
+import org.utbot.testcheckers.eq
+import org.utbot.tests.infrastructure.Compilation
+import org.utbot.tests.infrastructure.UtValueTestCaseChecker
+
+// TODO failed Kotlin tests execution with non-nullable expected field
+class ClassWithPrivateMutableFieldOfPrivateTypeTest : UtValueTestCaseChecker(
+    testClass = ClassWithPrivateMutableFieldOfPrivateType::class,
+    testCodeGeneration = true,
+    languagePipelines = listOf(
+        CodeGenerationLanguageLastStage(CodegenLanguage.JAVA),
+        CodeGenerationLanguageLastStage(CodegenLanguage.KOTLIN, Compilation)
+    )
+) {
+    @Test
+    fun testChangePrivateMutableFieldWithPrivateType() {
+        checkAllMutationsWithThis(
+            ClassWithPrivateMutableFieldOfPrivateType::changePrivateMutableFieldWithPrivateType,
+            eq(1),
+            { thisBefore, _, thisAfter, _, r ->
+                val privateMutableField = FieldId(
+                    ClassWithPrivateMutableFieldOfPrivateType::class.id,
+                    "privateMutableField"
+                ).jField
+
+                val (privateFieldBeforeValue, privateFieldAfterValue) = privateMutableField.withAccessibility {
+                    privateMutableField.get(thisBefore) to privateMutableField.get(thisAfter)
+                }
+
+                privateFieldBeforeValue == null && privateFieldAfterValue != null && r == 0
+            }
+        )
+    }
+}
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgFieldStateManager.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgFieldStateManager.kt
@@ -28,11 +28,7 @@ import org.utbot.framework.fields.ModifiedFields
 import org.utbot.framework.fields.StateModificationInfo
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.UtSymbolicExecution
-import org.utbot.framework.plugin.api.util.hasField
-import org.utbot.framework.plugin.api.util.id
-import org.utbot.framework.plugin.api.util.isArray
-import org.utbot.framework.plugin.api.util.isRefType
-import org.utbot.framework.plugin.api.util.objectClassId
+import org.utbot.framework.plugin.api.util.*
 import org.utbot.framework.util.hasThisInstance
 import org.utbot.fuzzer.UtFuzzedExecution
 import java.lang.reflect.Array
@@ -141,7 +137,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
         emptyLineIfNeeded()
         val fields = when (state) {
             FieldState.INITIAL -> modifiedFields
-                    .filter { it.path.elements.isNotEmpty() && it.path.fieldType.isRefType }
+                    .filter { it.path.elements.isNotEmpty() && !it.path.fieldType.isPrimitive }
                     .filter { needExpectedDeclaration(it.after) }
             FieldState.FINAL -> modifiedFields
         }
@@ -229,7 +225,9 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
             if (index > path.lastIndex) return@generateSequence null
             val passedPath = FieldPath(path.subList(0, index + 1))
             val name = if (index == path.lastIndex) customName else getFieldVariableName(prev, passedPath)
-            val expression = when (val newElement = path[index++]) {
+
+            val newElement = path[index++]
+            val expression = when (newElement) {
                 is FieldAccess -> {
                     val fieldId = newElement.field
                     utilsClassId[getFieldValue](prev, fieldId.declaringClass.name, fieldId.name)
@@ -238,7 +236,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
                     Array::class.id[getArrayElement](prev, newElement.index)
                 }
             }
-            newVar(objectClassId, name) { expression }
+            newVar(newElement.type, name) { expression }
         }.last()
     }
 
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt
@@ -58,11 +58,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
 import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
 import org.utbot.framework.codegen.model.tree.CgTestMethod
 import org.utbot.framework.codegen.model.tree.CgTestMethodType
-import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
-import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
-import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
-import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
-import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
+import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
 import org.utbot.framework.codegen.model.tree.CgTryCatch
 import org.utbot.framework.codegen.model.tree.CgTypeCast
 import org.utbot.framework.codegen.model.tree.CgValue
@@ -269,15 +265,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                             assertEquality(expected, actual)
                         }
                     }
-                    .onFailure { exception ->
-                        processExecutionFailure(currentExecution, exception)
-                    }
+                    .onFailure { exception -> processExecutionFailure(exception) }
             }
             else -> {} // TODO: check this specific case
         }
     }
 
-    private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
+    private fun processExecutionFailure(exception: Throwable) {
         val methodInvocationBlock = {
             with(currentExecutable) {
                 when (this) {
@@ -288,42 +282,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
             }
         }
 
-        if (shouldTestPassWithException(execution, exception)) {
-            testFrameworkManager.expectException(exception::class.id) {
-                methodInvocationBlock()
-            }
-            methodType = SUCCESSFUL
-
-            return
-        }
-
-        if (shouldTestPassWithTimeoutException(execution, exception)) {
-            writeWarningAboutTimeoutExceeding()
-            testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
-                methodInvocationBlock()
+        when (methodType) {
+            SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
+            PASSED_EXCEPTION -> {
+                testFrameworkManager.expectException(exception::class.id) {
+                    methodInvocationBlock()
+                }
             }
-            methodType = TIMEOUT
-
-            return
-        }
-
-        when (exception) {
-            is ConcreteExecutionFailureException -> {
-                methodType = CRASH
-                writeWarningAboutCrash()
+            TIMEOUT -> {
+                writeWarningAboutTimeoutExceeding()
+                testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
+                    methodInvocationBlock()
+                }
             }
-            is AccessControlException -> {
-                methodType = CRASH
-                writeWarningAboutFailureTest(exception)
-                return
+            CRASH -> when (exception) {
+                is ConcreteExecutionFailureException -> {
+                    writeWarningAboutCrash()
+                    methodInvocationBlock()
+                }
+                is AccessControlException -> {
+                    // exception from sandbox
+                    writeWarningAboutFailureTest(exception)
+                }
+                else -> error("Unexpected crash suite for failing execution with $exception exception")
             }
-            else -> {
-                methodType = FAILING
+            FAILING -> {
                 writeWarningAboutFailureTest(exception)
+                methodInvocationBlock()
             }
+            PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
         }
-
-        methodInvocationBlock()
     }
 
     private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1157,9 +1145,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                     constructorCall(*methodArguments.toTypedArray())
                 }
             }
-            .onFailure { exception ->
-                processExecutionFailure(currentExecution, exception)
-            }
+            .onFailure { exception -> processExecutionFailure(exception) }
     }
 
     /**
@@ -1258,11 +1244,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                         val name = paramNames[executableId]?.get(index)
                         methodArguments += variableConstructor.getOrCreateVariable(param, name)
                     }
-                    fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
+
+                    if (requiresFieldStateAssertions()) {
+                        // we should generate field assertions only for successful tests
+                        // that does not break the current test execution after invocation of method under test
+                        fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
+                    }
+
                     recordActualResult()
                     generateResultAssertions()
-                    fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
-                    generateFieldStateAssertions()
+
+                    if (requiresFieldStateAssertions()) {
+                        // we should generate field assertions only for successful tests
+                        // that does not break the current test execution after invocation of method under test
+                        fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
+                        generateFieldStateAssertions()
+                    }
                 }
 
                 if (statics.isNotEmpty()) {
@@ -1310,6 +1307,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
             }
         }
 
+    private fun requiresFieldStateAssertions(): Boolean =
+        !methodType.isThrowing ||
+                (methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
+
     private val expectedResultVarName = "expectedResult"
     private val expectedErrorVarName = "expectedError"
 
@@ -1337,7 +1338,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
                     substituteStaticFields(statics, isParametrized = true)
 
                     // build this instance
-                    thisInstance = genericExecution.stateBefore.thisInstance?.let {
+                    thisInstance =
+                        genericExecution.stateBefore.thisInstance?.let {
                         variableConstructor.getOrCreateVariable(it)
                     }
 
@@ -1553,6 +1555,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
     private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
         clearTestMethodScope()
         currentExecution = execution
+        determineExecutionType()
         statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
         return try {
             block()
@@ -1619,6 +1622,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
         testSet.executions.any { it.result is UtExecutionFailure }
 
 
+    /**
+     * Determines [CgTestMethodType] for current execution according to its success or failure.
+     */
+    private fun determineExecutionType() {
+        val currentExecution = currentExecution!!
+
+        currentExecution.result
+            .onSuccess { methodType = SUCCESSFUL }
+            .onFailure { exception ->
+                methodType = when {
+                    shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
+                    shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
+                    else -> when (exception) {
+                        is ConcreteExecutionFailureException -> CRASH
+                        is AccessControlException -> CRASH // exception from sandbox
+                        else -> FAILING
+                    }
+                }
+            }
+    }
+
     private fun testMethod(
         methodName: String,
         displayName: String?,
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt
@@ -86,6 +86,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
 
     abstract val annotationForOuterClasses: CgAnnotation?
 
+    /**
+     * Determines whether appearance of expected exception in test method breaks current test execution or not.
+     */
+    abstract val isExpectedExceptionExecutionBreaking: Boolean
+
     protected open val timeoutArgumentName: String = "timeout"
 
     open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -246,6 +251,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
     override val annotationForOuterClasses: CgAnnotation?
         get() = null
 
+    override val isExpectedExceptionExecutionBreaking: Boolean = false
+
     override val timeoutArgumentName: String = "timeOut"
 
     private val assertThrows: BuiltinMethodId
@@ -403,6 +410,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
             )
         }
 
+    override val isExpectedExceptionExecutionBreaking: Boolean = true
+
     override fun expectException(exception: ClassId, block: () -> Unit) {
         require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
 
@@ -466,6 +475,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
     override val annotationForOuterClasses: CgAnnotation?
         get() = null
 
+    override val isExpectedExceptionExecutionBreaking: Boolean = false
+
     private val assertThrows: BuiltinMethodId
         get() {
             require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestsGenerationReport.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestsGenerationReport.kt
@@ -4,6 +4,7 @@ import org.utbot.common.appendHtmlLine
 import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
 import org.utbot.framework.codegen.model.tree.CgTestMethod
 import org.utbot.framework.codegen.model.tree.CgTestMethodType
+import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
 import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.util.kClass
 import kotlin.reflect.KClass
@@ -57,11 +58,11 @@ data class TestsGenerationReport(
 
             testMethods.forEach {
                 when (it.type) {
-                    CgTestMethodType.SUCCESSFUL -> updateExecutions(it, successfulExecutions)
-                    CgTestMethodType.FAILING -> updateExecutions(it, failedExecutions)
-                    CgTestMethodType.TIMEOUT -> updateExecutions(it, timeoutExecutions)
-                    CgTestMethodType.CRASH -> updateExecutions(it, crashExecutions)
-                    CgTestMethodType.PARAMETRIZED -> {
+                    SUCCESSFUL, PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
+                    FAILING -> updateExecutions(it, failedExecutions)
+                    TIMEOUT -> updateExecutions(it, timeoutExecutions)
+                    CRASH -> updateExecutions(it, crashExecutions)
+                    PARAMETRIZED -> {
                         // Parametrized tests are not supported in the tests report yet
                         // TODO JIRA:1507
                     }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt
@@ -335,12 +335,13 @@ class CgParameterizedTestDataProviderMethod(
     override val requiredFields: List<CgParameterDeclaration> = emptyList()
 }
 
-enum class CgTestMethodType(val displayName: String) {
-    SUCCESSFUL("Successful tests"),
-    FAILING("Failing tests (with exceptions)"),
-    TIMEOUT("Failing tests (with timeout)"),
-    CRASH("Possibly crashing tests"),
-    PARAMETRIZED("Parametrized tests");
+enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
+    SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
+    PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
+    FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
+    TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
+    CRASH(displayName = "Possibly crashing tests", isThrowing = true),
+    PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
 
     override fun toString(): String = displayName
 }
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/visitor/CgAbstractRenderer.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/visitor/CgAbstractRenderer.kt
@@ -326,7 +326,7 @@ internal abstract class CgAbstractRenderer(
         if (lines.isEmpty()) return
 
         if (lines.size == 1) {
-            print("/* ${lines.first()} */")
+            println("/* ${lines.first()} */")
             return
         }
 
diff --git a/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/UtValueTestCaseChecker.kt b/utbot-framework/src/main/kotlin/org/utbot/tests/infrastructure/UtValueTestCaseChecker.kt
diff --git a/utbot-sample/src/main/java/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateType.java b/utbot-sample/src/main/java/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateType.java

Original file line number	Diff line number	Diff line change
`@@ -326,7 +326,7 @@ internal abstract class CgAbstractRenderer(`
`326`	`326`	`if (lines.isEmpty()) return`
`327`	`327`
`328`	`328`	`if (lines.size == 1) {`
`329`		`- print("/* ${lines.first()} */")`
	`329`	`+ println("/* ${lines.first()} */")`
`330`	`330`	`return`
`331`	`331`	`}`
`332`	`332`