Add support for Artificial Errors in Summaries #1739 (#1785)

sofurihafe · web-flow · commit 196dc2a3f992 · 2023-02-13T10:33:44.000Z
Support Artificial Errors in summaries
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/ArtificialErrors.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/ArtificialErrors.kt
@@ -1,4 +1,4 @@
-package org.utbot.engine
+package org.utbot.framework.plugin.api
 
 /**
  * Represents an error that may be detected or not
@@ -15,4 +15,9 @@ sealed class ArtificialError(message: String): Error(message)
  *
  * See [TraversalContext.intOverflowCheck] for more details.
  */
-class OverflowDetectionError(message: String): ArtificialError(message)
+class OverflowDetectionError(message: String): ArtificialError(message)
+
+fun ArtificialError.getPrettyName(): String =
+    when (this) {
+        is OverflowDetectionError -> "Overflow"
+    }
diff --git a/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt b/utbot-framework-test/src/test/kotlin/org/utbot/examples/math/OverflowAsErrorTest.kt
@@ -2,7 +2,7 @@ package org.utbot.examples.math
 
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
-import org.utbot.engine.OverflowDetectionError
+import org.utbot.framework.plugin.api.OverflowDetectionError
 import org.utbot.examples.algorithms.Sort
 import org.utbot.framework.plugin.api.CodegenLanguage
 import org.utbot.testcheckers.eq
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/Resolver.kt
@@ -90,6 +90,7 @@ import kotlin.math.max
 import kotlin.math.min
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.persistentSetOf
+import org.utbot.framework.plugin.api.OverflowDetectionError
 import org.utbot.engine.types.CLASS_REF_CLASSNAME
 import org.utbot.engine.types.CLASS_REF_CLASS_ID
 import org.utbot.engine.types.CLASS_REF_NUM_DIMENSIONS_DESCRIPTOR
diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/Traverser.kt
@@ -8,6 +8,8 @@ import kotlinx.collections.immutable.toPersistentList
 import kotlinx.collections.immutable.toPersistentMap
 import kotlinx.collections.immutable.toPersistentSet
 import mu.KotlinLogging
+import org.utbot.framework.plugin.api.ArtificialError
+import org.utbot.framework.plugin.api.OverflowDetectionError
 import org.utbot.common.WorkaroundReason.HACK
 import org.utbot.framework.UtSettings.ignoreStaticsFromTrustedLibraries
 import org.utbot.common.WorkaroundReason.IGNORE_STATICS_FROM_TRUSTED_LIBRARIES
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt
@@ -3,8 +3,8 @@ package org.utbot.framework.codegen.tree
 import org.utbot.common.WorkaroundReason
 import org.utbot.common.isStatic
 import org.utbot.common.workaround
-import org.utbot.engine.ArtificialError
 import org.utbot.framework.UtSettings
+import org.utbot.framework.plugin.api.ArtificialError
 import org.utbot.framework.assemble.assemble
 import org.utbot.framework.codegen.domain.ForceStaticMocking
 import org.utbot.framework.codegen.domain.ParametrizedTestSource
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/util/SootUtils.kt
@@ -2,7 +2,6 @@ package org.utbot.framework.util
 
 import org.utbot.common.FileUtil
 import org.utbot.engine.jimpleBody
-import org.utbot.engine.overrides.collections.UtLinkedList
 import org.utbot.engine.pureJavaSignature
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.ExecutableId
@@ -208,7 +207,7 @@ private val classesToLoad = arrayOf(
     org.utbot.engine.overrides.stream.IntStream::class,
     org.utbot.engine.overrides.stream.LongStream::class,
     org.utbot.engine.overrides.stream.DoubleStream::class,
-    org.utbot.engine.OverflowDetectionError::class,
+    org.utbot.framework.plugin.api.OverflowDetectionError::class,
 ).map { it.java }.toTypedArray()
 
 private const val UTBOT_PACKAGE_PREFIX = "org.utbot"
diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryOverflowExamples.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryOverflowExamples.kt
@@ -0,0 +1,54 @@
+package math
+
+import examples.CustomJavaDocTagsEnabler
+import examples.SummaryTestCaseGeneratorTest
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.utbot.examples.math.OverflowExamples
+import org.utbot.framework.plugin.api.MockStrategyApi
+import org.utbot.testcheckers.withTreatingOverflowAsError
+import org.utbot.testing.DoNotCalculate
+
+@ExtendWith(CustomJavaDocTagsEnabler::class)
+class SummaryOverflowExamples : SummaryTestCaseGeneratorTest(
+    OverflowExamples::class
+) {
+    @Test
+    fun testShortMulOverflow() {
+        val summary1 = "@utbot.classUnderTest {@link OverflowExamples}\n" +
+                "@utbot.methodUnderTest {@link org.utbot.examples.math.OverflowExamples#shortMulOverflow(short,short)}\n" +
+                "@utbot.returnsFrom {@code return (short) (x * y);}\n"
+        val summary2 = "@utbot.classUnderTest {@link OverflowExamples}\n" +
+                "@utbot.methodUnderTest {@link org.utbot.examples.math.OverflowExamples#shortMulOverflow(short,short)}\n" +
+                "@utbot.detectsSuspiciousBehavior in: return (short) (x * y);\n"
+
+        val methodName1 = "testShortMulOverflow_ReturnXy"
+        val methodName2 = "testShortMulOverflow_DetectOverflow"
+
+        val displayName1 = "-> return (short) (x * y)"
+        val displayName2 = "return (short) (x * y) : True -> DetectOverflow"
+
+        val summaryKeys = listOf(
+            summary1,
+            summary2
+        )
+
+        val displayNames = listOf(
+            displayName1,
+            displayName2
+        )
+
+        val methodNames = listOf(
+            methodName1,
+            methodName2
+        )
+
+        val method = OverflowExamples::shortMulOverflow
+        val mockStrategy = MockStrategyApi.NO_MOCKS
+        val coverage = DoNotCalculate
+
+        withTreatingOverflowAsError {
+            summaryCheck(method, mockStrategy, coverage, summaryKeys, methodNames, displayNames)
+        }
+    }
+}
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/classic/symbolic/SimpleCommentBuilder.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/classic/symbolic/SimpleCommentBuilder.kt
@@ -7,6 +7,7 @@ import com.github.javaparser.ast.stmt.IfStmt
 import com.github.javaparser.ast.stmt.Statement
 import com.github.javaparser.ast.stmt.SwitchStmt
 import com.github.javaparser.ast.stmt.ThrowStmt
+import org.utbot.framework.plugin.api.ArtificialError
 import org.utbot.framework.plugin.api.InstrumentedProcessDeathException
 import org.utbot.framework.plugin.api.DocPreTagStatement
 import org.utbot.framework.plugin.api.DocRegularStmt
@@ -67,7 +68,11 @@ open class SimpleCommentBuilder(
         traceTag.result.exceptionOrNull()?.let {
             val exceptionName = it.javaClass.simpleName
             val reason = findExceptionReason(currentMethod, it)
-            root.exceptionThrow = "$exceptionName $reason"
+
+            when (it) {
+                is ArtificialError -> root.detectedError = reason
+                else -> root.exceptionThrow = "$exceptionName $reason"
+            }
         }
     }
 
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/classic/symbolic/SimpleSentenceBlock.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/classic/symbolic/SimpleSentenceBlock.kt
@@ -19,6 +19,7 @@ class SimpleSentenceBlock(val stringTemplates: StringsTemplatesInterface) {
     var notExecutedIterations: List<IterationDescription>? = null
     var recursion: Pair<String, SimpleSentenceBlock>? = null
     var exceptionThrow: String? = null
+    var detectedError: String? = null
     var nextBlock: SimpleSentenceBlock? = null
 
     val stmtTexts = mutableListOf<StmtDescription>()
@@ -122,6 +123,15 @@ class SimpleSentenceBlock(val stringTemplates: StringsTemplatesInterface) {
             result += NEW_LINE
         }
 
+        detectedError?.let {
+            if (restartSentence) {
+                result += stringTemplates.sentenceBeginning + " "
+                restartSentence = false
+            }
+            result += stringTemplates.suspiciousBehaviorDetectedSentence.format(it)
+            result += NEW_LINE
+        }
+
         return result
     }
 
@@ -238,6 +248,16 @@ class SimpleSentenceBlock(val stringTemplates: StringsTemplatesInterface) {
             docStmts += DocRegularStmt(NEW_LINE)
         }
 
+        detectedError?.let {
+            docStmts += DocRegularStmt(NEW_LINE)
+            if (restartSentence) {
+                docStmts += DocRegularStmt(stringTemplates.sentenceBeginning + " ")
+                restartSentence = false
+            }
+            docStmts += DocRegularStmt(stringTemplates.suspiciousBehaviorDetectedSentence.format(it))
+            docStmts += DocRegularStmt(NEW_LINE)
+        }
+
         return docStmts
     }
 
@@ -323,6 +343,7 @@ class SimpleSentenceBlock(val stringTemplates: StringsTemplatesInterface) {
         if (nextBlock?.isEmpty() == false) return false
         if (notExecutedIterations.isNullOrEmpty().not()) return false
         exceptionThrow?.let { return false }
+        detectedError?.let { return false }
         if (invokeSentenceBlock != null) return false
         return true
     }
@@ -478,6 +499,7 @@ interface StringsTemplatesInterface {
     val noIteration: String
     val codeSentence: String
     val throwSentence: String
+    val suspiciousBehaviorDetectedSentence: String
 
     val conditionLine: String
     val returnLine: String
@@ -498,6 +520,7 @@ open class StringsTemplatesSingular : StringsTemplatesInterface {
     override val noIteration: String = "does not iterate"
     override val codeSentence: String = "$OPEN_BRACKET$AT_CODE %s$CLOSE_BRACKET"
     override val throwSentence: String = "throws %s"
+    override val suspiciousBehaviorDetectedSentence: String = "detects suspicious behavior %s"
 
     //used in Squashing
     override val conditionLine: String = "executes conditions:$NEW_LINE"
@@ -519,6 +542,7 @@ class StringsTemplatesPlural : StringsTemplatesSingular() {
     override val noIteration: String = "does not iterate"
     override val codeSentence: String = "$OPEN_BRACKET$AT_CODE %s$CLOSE_BRACKET"
     override val throwSentence: String = "throw %s"
+    override val suspiciousBehaviorDetectedSentence: String = "detect suspicious behavior %s"
 
     //used in Squashing
     override val conditionLine: String = "execute conditions:$NEW_LINE"
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/CustomJavaDocTagProvider.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/CustomJavaDocTagProvider.kt
@@ -24,6 +24,7 @@ class CustomJavaDocTagProvider {
             CustomJavaDocTag.ReturnsFrom,
             CustomJavaDocTag.CaughtException,
             CustomJavaDocTag.ThrowsException,
+            CustomJavaDocTag.DetectsSuspiciousBehavior,
         )
 }
 
@@ -73,6 +74,9 @@ sealed class CustomJavaDocTag(
     object ThrowsException :
         CustomJavaDocTag("utbot.throwsException", "Throws exception", CustomJavaDocComment::throwsException, null)
 
+    object DetectsSuspiciousBehavior :
+        CustomJavaDocTag("utbot.detectsSuspiciousBehavior", "Detects suspicious behavior", CustomJavaDocComment::detectsSuspiciousBehavior, null)
+
     fun generateDocStatement(comment: CustomJavaDocComment): DocRegularStmt? =
         when (val value = valueRetriever.invoke(comment)) {
             is String -> value.takeIf { it.isNotEmpty() }?.let {
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/symbolic/CustomJavaDocComment.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/symbolic/CustomJavaDocComment.kt
@@ -18,5 +18,6 @@ data class CustomJavaDocComment(
     var returnsFrom: String = EMPTY_STRING,
     var countedReturn: String = EMPTY_STRING,
     var caughtException: String = EMPTY_STRING,
-    var throwsException: String = EMPTY_STRING
+    var throwsException: String = EMPTY_STRING,
+    var detectsSuspiciousBehavior: String = EMPTY_STRING
 )
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/symbolic/CustomJavaDocCommentBuilder.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/comment/customtags/symbolic/CustomJavaDocCommentBuilder.kt
@@ -1,5 +1,6 @@
 package org.utbot.summary.comment.customtags.symbolic
 
+import org.utbot.framework.plugin.api.ArtificialError
 import org.utbot.framework.plugin.api.DocCustomTagStatement
 import org.utbot.framework.plugin.api.DocStatement
 import org.utbot.framework.plugin.api.exceptionOrNull
@@ -54,8 +55,12 @@ class CustomJavaDocCommentBuilder(
         if (thrownException != null) {
             val exceptionName = thrownException.javaClass.name
             val reason = findExceptionReason(currentMethod, thrownException)
+                .replace(CARRIAGE_RETURN, "")
 
-            comment.throwsException = "{@link $exceptionName} $reason".replace(CARRIAGE_RETURN, "")
+            when (thrownException) {
+                is ArtificialError -> comment.detectsSuspiciousBehavior = reason
+                else -> comment.throwsException = "{@link $exceptionName} $reason"
+            }
         }
 
         if (rootSentenceBlock.recursion != null) {
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/name/NameUtil.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/name/NameUtil.kt
@@ -1,6 +1,8 @@
 package org.utbot.summary.name
 
+import org.utbot.framework.plugin.api.ArtificialError
 import org.utbot.framework.plugin.api.Step
+import org.utbot.framework.plugin.api.getPrettyName
 import org.utbot.summary.tag.UniquenessTag
 import soot.SootMethod
 
@@ -49,7 +51,7 @@ data class TestNameDescription(
 }
 
 enum class NameType {
-    Condition, Return, Invoke, SwitchCase, CaughtException, NoIteration, ThrowsException, StartIteration
+    Condition, Return, Invoke, SwitchCase, CaughtException, NoIteration, ThrowsException, StartIteration, ArtificialError
 }
 
 data class DisplayNameCandidate(val name: String, val uniquenessTag: UniquenessTag, val index: Int)
@@ -61,4 +63,21 @@ fun List<TestNameDescription>.returnsToUnique() = this.map {
     } else {
         it
     }
+}
+
+fun buildNameFromThrowable(exception: Throwable): String? {
+    val exceptionName = exception::class.simpleName
+
+    if (exceptionName.isNullOrEmpty()) return null
+    return when (exception) {
+        is ArtificialError -> "Detect${exception.getPrettyName()}"
+        else -> "Throw$exceptionName"
+    }
+}
+
+fun getThrowableNameType(exception: Throwable): NameType {
+    return when (exception) {
+        is ArtificialError -> NameType.ArtificialError
+        else -> NameType.ThrowsException
+    }
 }
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/name/SimpleNameBuilder.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/name/SimpleNameBuilder.kt
@@ -146,6 +146,13 @@ class SimpleNameBuilder(
     }
 
     private fun fromNameDescriptionToCandidateSimpleName(nameDescription: TestNameDescription): DisplayNameCandidate? {
+        if (nameDescription.nameType == NameType.ArtificialError) {
+            return DisplayNameCandidate(
+                nameDescription.name,
+                nameDescription.uniquenessTag,
+                traceTag.path.size + 1
+            )
+        }
         if (nameDescription.nameType == NameType.ThrowsException) {
             return DisplayNameCandidate(
                 nameDescription.name,
@@ -268,14 +275,18 @@ class SimpleNameBuilder(
      * (for example, [InstrumentedProcessDeathException]).
      */
     private fun exceptionThrow(testNames: MutableList<TestNameDescription>) {
-        val throwsException = traceTag.result.exceptionOrNull()?.let { it::class.simpleName }
-        if (!(throwsException.isNullOrEmpty() || traceTag.path.isEmpty())) {
+        val exception = traceTag.result.exceptionOrNull() ?: return
+        val name = buildNameFromThrowable(exception)
+
+        if (name != null && traceTag.path.isNotEmpty()) {
+            val nameType = getThrowableNameType(exception)
+
             testNames.add(TestNameDescription(
-                "Throw$throwsException",
+                name,
                 testNames.maxOfOrNull { it.depth } ?: 0,
                 testNames.maxOfOrNull { it.line } ?: 0,
                 UniquenessTag.Unique,
-                NameType.ThrowsException,
+                nameType,
                 testNames.maxOfOrNull { it.index } ?: 0,
                 traceTag.path.last(),
                 methodUnderTest
