Add fuzzer executions as standard tests in parameterized tests

Author: sofurihafe

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/CgMethodTestSet.kt

@@ -32,11 +32,55 @@ data class CgMethodTestSet private constructor(
         executions = from.executions
     }
 
+    fun prepareForParameterizedTestGeneration(): List<CgMethodTestSet> {
+        val testSetList = mutableListOf<CgMethodTestSet>()
+
+        // Mocks are not supported in parametrized tests, so we exclude them
+        val testSetWithoutMocking = this.excludeExecutionsWithMocking()
+        for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
+            for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
+                testSetList += splitByChangedStaticsTestSet
+            }
+        }
+
+        return testSetList
+    }
+
+    fun prepareFuzzedExecutions(): CgMethodTestSet {
+        val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
+
+        return substituteExecutions(fuzzedExecutions)
+    }
+
+    /**
+     * Finds a [ClassId] of all result models in executions.
+     *
+     * Tries to find a unique result type in testSets or
+     * gets executable return type.
+     */
+    fun resultType(): ClassId {
+        return when (executableId.returnType) {
+            voidClassId -> executableId.returnType
+            else -> {
+                val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
+                if (successfulExecutions.isNotEmpty()) {
+                    successfulExecutions
+                        .map { (it.result as UtExecutionSuccess).model.classId }
+                        .distinct()
+                        .singleOrNull()
+                        ?: executableId.returnType
+                } else {
+                    executableId.returnType
+                }
+            }
+        }
+    }
+
     /**
      * Splits [CgMethodTestSet] into separate test sets having
      * unique result model [ClassId] in each subset.
      */
-    fun splitExecutionsByResult() : List<CgMethodTestSet> {
+    private fun splitExecutionsByResult() : List<CgMethodTestSet> {
         val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
         val failureExecutions = executions.filter { it.result is UtExecutionFailure }
 
@@ -60,47 +104,22 @@ data class CgMethodTestSet private constructor(
      *
      * A separate test set is created for each combination of modified statics.
      */
-    fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
+    private fun splitExecutionsByChangedStatics(): List<CgMethodTestSet> {
         val executionsByStaticsUsage: Map<Set<FieldId>, List<UtExecution>> =
             executions.groupBy { it.stateBefore.statics.keys }
 
         return executionsByStaticsUsage.map { (_, executions) -> substituteExecutions(executions) }
     }
 
     /*
-    * Excludes executions with mocking from [CgMethodTestSet].
+    * Excludes fuzzed executions and executions with mocking from [CgMethodTestSet].
     * */
-    fun excludeExecutionsWithMocking(): CgMethodTestSet {
-        val fuzzedExecutions = executions.filterIsInstance<UtFuzzedExecution>()
+    private fun excludeExecutionsWithMocking(): CgMethodTestSet {
         val symbolicExecutionsWithoutMocking = executions
             .filterIsInstance<UtSymbolicExecution>()
             .filter { !it.containsMocking }
 
-        return substituteExecutions(symbolicExecutionsWithoutMocking + fuzzedExecutions)
-    }
-
-    /**
-     * Finds a [ClassId] of all result models in executions.
-     *
-     * Tries to find a unique result type in testSets or
-     * gets executable return type.
-     */
-    fun resultType(): ClassId {
-        return when (executableId.returnType) {
-            voidClassId -> executableId.returnType
-            else -> {
-                val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
-                if (successfulExecutions.isNotEmpty()) {
-                    successfulExecutions
-                        .map { (it.result as UtExecutionSuccess).model.classId }
-                        .distinct()
-                        .singleOrNull()
-                        ?: executableId.returnType
-                } else {
-                    executableId.returnType
-                }
-            }
-        }
+        return substituteExecutions(symbolicExecutionsWithoutMocking)
     }
 
     private fun substituteExecutions(newExecutions: List<UtExecution>): CgMethodTestSet =

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

@@ -37,7 +37,6 @@ import org.utbot.framework.codegen.model.tree.buildTestClassBody
 import org.utbot.framework.codegen.model.tree.buildTestClassFile
 import org.utbot.framework.codegen.model.visitor.importUtilMethodDependencies
 import org.utbot.framework.plugin.api.ClassId
-import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.MethodId
 import org.utbot.framework.plugin.api.UtExecutionSuccess
 import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -140,45 +139,18 @@ internal class CgTestClassConstructor(val context: CgContext) :
             .filter { it.result is UtExecutionSuccess }
             .map { (it.result as UtExecutionSuccess).model }
 
-        val (methodUnderTest, _, _, clustersInfo) = testSet
         val regions = mutableListOf<CgRegion<CgMethod>>()
-        val requiredFields = mutableListOf<CgParameterDeclaration>()
-
-        when (context.parametrizedTestSource) {
-            ParametrizedTestSource.DO_NOT_PARAMETRIZE -> {
-                for ((clusterSummary, executionIndices) in clustersInfo) {
-                    val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
-                    emptyLineIfNeeded()
-                    for (i in executionIndices) {
-                        runCatching {
-                            currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
-                        }.onFailure { e -> processFailure(testSet, e) }
-                    }
-                    val clusterHeader = clusterSummary?.header
-                    val clusterContent = clusterSummary?.content
-                        ?.split('\n')
-                        ?.let { CgTripleSlashMultilineComment(it) }
-                    regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
 
-                    testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
-                }
-            }
-            ParametrizedTestSource.PARAMETRIZE -> {
-                // Mocks are not supported in parametrized tests, we should exclude them
-                val testSetWithoutMocking = testSet.excludeExecutionsWithMocking()
-
-                for (splitByExecutionTestSet in testSetWithoutMocking.splitExecutionsByResult()) {
-                    for (splitByChangedStaticsTestSet in splitByExecutionTestSet.splitExecutionsByChangedStatics()) {
-                        createParametrizedTestAndDataProvider(
-                            splitByChangedStaticsTestSet,
-                            requiredFields,
-                            regions,
-                            methodUnderTest
-                        )
-                    }
-                }
+        runCatching {
+            when (context.parametrizedTestSource) {
+                ParametrizedTestSource.DO_NOT_PARAMETRIZE -> createTest(testSet, regions)
+                ParametrizedTestSource.PARAMETRIZE ->
+                    createParametrizedTestAndDataProvider(
+                        testSet,
+                        regions
+                    )
             }
-        }
+        }.onFailure { e -> processFailure(testSet, e) }
 
         val errors = testSet.allErrors
         if (errors.isNotEmpty()) {
@@ -195,29 +167,61 @@ internal class CgTestClassConstructor(val context: CgContext) :
             .merge(failure.description, 1, Int::plus)
     }
 
+    private fun createTest(
+        testSet: CgMethodTestSet,
+        regions: MutableList<CgRegion<CgMethod>>
+    ) {
+        val (methodUnderTest, _, _, clustersInfo) = testSet
+
+        for ((clusterSummary, executionIndices) in clustersInfo) {
+            val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
+            emptyLineIfNeeded()
+            for (i in executionIndices) {
+                currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
+            }
+            val clusterHeader = clusterSummary?.header
+            val clusterContent = clusterSummary?.content
+                ?.split('\n')
+                ?.let { CgTripleSlashMultilineComment(it) }
+            regions += CgTestMethodCluster(clusterHeader, clusterContent, currentTestCaseTestMethods)
+
+            testsGenerationReport.addTestsByType(testSet, currentTestCaseTestMethods)
+        }
+    }
+
     private fun createParametrizedTestAndDataProvider(
         testSet: CgMethodTestSet,
-        requiredFields: MutableList<CgParameterDeclaration>,
-        regions: MutableList<CgRegion<CgMethod>>,
-        methodUnderTest: ExecutableId,
+        regions: MutableList<CgRegion<CgMethod>>
     ) {
-        runCatching {
-            val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
+        val (methodUnderTest, _, _, _) = testSet
 
-            val parameterizedTestMethod =
-                methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
+        for (preparedTestSet in testSet.prepareForParameterizedTestGeneration()) {
+            val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(preparedTestSet.executableId)
 
-            requiredFields += parameterizedTestMethod.requiredFields
+            val parameterizedTestMethod =
+                methodConstructor.createParameterizedTestMethod(preparedTestSet, dataProviderMethodName)
 
             testFrameworkManager.addDataProvider(
-                methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
+                methodConstructor.createParameterizedTestDataProvider(preparedTestSet, dataProviderMethodName)
             )
 
             regions += CgSimpleRegion(
                 "Parameterized test for method ${methodUnderTest.humanReadableName}",
                 listOf(parameterizedTestMethod),
             )
-        }.onFailure { error -> processFailure(testSet, error) }
+        }
+
+        // We cannot track mocking in fuzzed executions, so we generate standard tests for them
+        // [https://github.com/UnitTestBot/UTBotJava/issues/1137]
+        val testCaseTestMethods = mutableListOf<CgTestMethod>()
+        for (execution in testSet.prepareFuzzedExecutions().executions) {
+            testCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, execution)
+        }
+
+        regions += CgSimpleRegion(
+            "FUZZER: EXECUTIONS for method ${methodUnderTest.humanReadableName}",
+            testCaseTestMethods,
+        )
     }
 
     /**