Skip to content

Commit 304b3ba

Browse files
amandelpieamandelpie
authored
Unified error and exception clusters for all UtExecutions (#1250)
* Unified error and exception clusters for all UtExecutions * Fixed JavaDocs
1 parent 742e03d commit 304b3ba

File tree

3 files changed

+91
-95
lines changed

3 files changed

+91
-95
lines changed

‎utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt

Lines changed: 20 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,6 @@ import java.io.File
2323
import java.nio.file.Path
2424
import java.nio.file.Paths
2525
import mu.KotlinLogging
26-
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
27-
import org.utbot.framework.plugin.api.UtExecutionSuccess
28-
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
29-
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
30-
import org.utbot.framework.plugin.api.UtOverflowFailure
31-
import org.utbot.framework.plugin.api.UtSandboxFailure
32-
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
33-
import org.utbot.framework.plugin.api.UtTimeoutException
34-
import org.utbot.framework.plugin.api.util.humanReadableName
3526
import org.utbot.framework.plugin.api.util.jClass
3627
import org.utbot.fuzzer.FuzzedMethodDescription
3728
import org.utbot.fuzzer.FuzzedValue
@@ -222,9 +213,8 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
222213
testSet: UtMethodTestSet
223214
): List<UtExecutionCluster> {
224215
val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
225-
val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
226-
val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
227-
val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
216+
val testSetWithFuzzedExecutions = prepareTestSetWithFuzzedExecutions(testSet)
217+
val executionsProducedByFuzzer = testSetWithFuzzedExecutions.executions as List<UtFuzzedExecution>
228218

229219
if (executionsProducedByFuzzer.isNotEmpty()) {
230220
executionsProducedByFuzzer.forEach { utExecution ->
@@ -239,44 +229,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
239229
)
240230
}.firstOrNull()
241231
} catch (t: Throwable) {
242-
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
232+
logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behaviour
243233
null
244234
}
245-
246235
utExecution.testMethodName = testMethodName?.testName
247236
utExecution.displayName = testMethodName?.displayName
248237
utExecution.summary = testMethodName?.javaDoc
249-
250-
when (utExecution.result) {
251-
is UtConcreteExecutionFailure,
252-
is UtExplicitlyThrownException,
253-
is UtImplicitlyThrownException,
254-
is UtOverflowFailure,
255-
is UtSandboxFailure,
256-
is UtTimeoutException,
257-
is UtStreamConsumingFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
258-
is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
259-
}
260238
}
261239

262-
if (successfulFuzzerExecutions.isNotEmpty()) {
263-
val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)
264-
265-
clustersToReturn.add(
266-
UtExecutionCluster(
267-
UtClusterInfo(clusterHeader, null),
268-
successfulFuzzerExecutions
269-
)
270-
)
271-
}
272-
273-
if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
274-
val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)
240+
val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions)
275241

242+
clusteredExecutions.forEach {
276243
clustersToReturn.add(
277244
UtExecutionCluster(
278-
UtClusterInfo(clusterHeader, null),
279-
unsuccessfulFuzzerExecutions
245+
UtClusterInfo(it.header),
246+
it.executions
280247
)
281248
)
282249
}
@@ -285,20 +252,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
285252
return clustersToReturn.toList()
286253
}
287254

288-
private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
289-
val commentPrefix = "FUZZER:"
290-
val commentPostfix = "for method ${testSet.method.humanReadableName}"
291-
292-
return "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix"
293-
}
294-
295-
private fun buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet: UtMethodTestSet): String {
296-
val commentPrefix = "FUZZER:"
297-
val commentPostfix = "for method ${testSet.method.humanReadableName}"
298-
299-
return "$commentPrefix ${ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS} $commentPostfix"
300-
}
301-
302255
/** Filter and copies executions with non-empty paths. */
303256
private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
304257
val executions =
@@ -314,6 +267,19 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
314267
)
315268
}
316269

270+
/** Filter and copies fuzzed executions. */
271+
private fun prepareTestSetWithFuzzedExecutions(testSet: UtMethodTestSet): UtMethodTestSet {
272+
val executions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
273+
274+
return UtMethodTestSet(
275+
method = testSet.method,
276+
executions = executions,
277+
jimpleBody = testSet.jimpleBody,
278+
errors = testSet.errors,
279+
clustersInfo = testSet.clustersInfo
280+
)
281+
}
282+
317283
/** Filter and copies executions with non-empty paths. */
318284
private fun prepareTestSetWithEmptyPaths(testSet: UtMethodTestSet): UtMethodTestSet {
319285
val executions =

‎utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt

Lines changed: 59 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,20 @@ package org.utbot.summary
22

33
import org.utbot.framework.plugin.api.Step
44
import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
5-
import org.utbot.framework.plugin.api.UtSymbolicExecution
5+
import org.utbot.framework.plugin.api.UtExecution
66
import org.utbot.framework.plugin.api.UtExecutionResult
77
import org.utbot.framework.plugin.api.UtExecutionSuccess
88
import org.utbot.framework.plugin.api.UtExplicitlyThrownException
99
import org.utbot.framework.plugin.api.UtImplicitlyThrownException
10-
import org.utbot.framework.plugin.api.UtOverflowFailure
1110
import org.utbot.framework.plugin.api.UtMethodTestSet
11+
import org.utbot.framework.plugin.api.UtOverflowFailure
1212
import org.utbot.framework.plugin.api.UtSandboxFailure
1313
import org.utbot.framework.plugin.api.UtStreamConsumingFailure
14+
import org.utbot.framework.plugin.api.UtSymbolicExecution
1415
import org.utbot.framework.plugin.api.UtTimeoutException
1516
import org.utbot.framework.plugin.api.util.humanReadableName
1617
import org.utbot.framework.plugin.api.util.isCheckedException
18+
import org.utbot.fuzzer.UtFuzzedExecution
1719
import org.utbot.summary.UtSummarySettings.MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING
1820
import org.utbot.summary.clustering.MatrixUniqueness
1921
import org.utbot.summary.clustering.SplitSteps
@@ -29,7 +31,7 @@ class TagGenerator {
2931

3032
if (clusteredExecutions.isNotEmpty()) {
3133
val listOfSplitSteps = clusteredExecutions.map {
32-
val mUniqueness = MatrixUniqueness(it.executions)
34+
val mUniqueness = MatrixUniqueness(it.executions as List<UtSymbolicExecution>)
3335
mUniqueness.splitSteps()
3436
}
3537

@@ -64,7 +66,7 @@ class TagGenerator {
6466
traceTagClusters.add(
6567
TraceTagCluster(
6668
cluster.header,
67-
generateExecutionTags(cluster.executions, splitSteps),
69+
generateExecutionTags(cluster.executions as List<UtSymbolicExecution>, splitSteps),
6870
TraceTagWithoutExecution(
6971
commonStepsInCluster.toList(),
7072
cluster.executions.first().result,
@@ -88,51 +90,67 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt
8890

8991

9092
/**
91-
* Splits executions into clusters
92-
* By default there is 5 types of clusters:
93-
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
94-
* These are split by the type of execution result
93+
* Splits executions with empty paths into clusters.
9594
*
96-
* @return clustered executions
95+
* @return clustered executions.
9796
*/
9897
fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionCluster> {
9998
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
10099
val clusters = mutableListOf<ExecutionCluster>()
101-
val commentPrefix = "OTHER:"
100+
val commentPrefix = "OTHER:"
102101
val commentPostfix = "for method ${testSet.method.humanReadableName}"
103102

104103
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
105104

106105
val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
107106
if (successfulExecutions.isNotEmpty()) {
108107
clusters += SuccessfulExecutionCluster(
109-
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
110-
successfulExecutions.toList())
108+
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
109+
successfulExecutions.toList()
110+
)
111111
}
112112

113-
clusters += grouped
114-
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
115-
.map { (suffixId, group) ->
116-
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
117-
}
113+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
114+
return clusters
115+
}
116+
117+
/**
118+
* Splits fuzzed executions into clusters.
119+
*
120+
* @return clustered executions.
121+
*/
122+
fun groupFuzzedExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
123+
val methodExecutions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
124+
val clusters = mutableListOf<ExecutionCluster>()
125+
val commentPrefix = "FUZZER:"
126+
val commentPostfix = "for method ${testSet.method.humanReadableName}"
127+
128+
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
129+
130+
val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
131+
if (successfulExecutions.isNotEmpty()) {
132+
clusters += SuccessfulExecutionCluster(
133+
"$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
134+
successfulExecutions.toList()
135+
)
136+
}
137+
138+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
118139
return clusters
119140
}
120141

121142
/**
122-
* Splits executions produced by symbolic execution engine into clusters
123-
* By default there is 5 types of clusters:
124-
* Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
125-
* These are split by the type of execution result
143+
* Splits symbolic executions into clusters.
126144
*
127-
* If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution
128-
* then clustering algorithm splits those into more clusters
145+
* If Success cluster has more than [MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING] execution
146+
* then clustering algorithm splits those into more clusters.
129147
*
130-
* @return clustered executions
148+
* @return clustered executions.
131149
*/
132150
private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
133151
val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
134152
val clusters = mutableListOf<ExecutionCluster>()
135-
val commentPrefix = "SYMBOLIC EXECUTION:"
153+
val commentPrefix = "SYMBOLIC EXECUTION:"
136154
val commentPostfix = "for method ${testSet.method.humanReadableName}"
137155

138156
val grouped = methodExecutions.groupBy { it.result.clusterKind() }
@@ -161,11 +179,21 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster
161179
}
162180
}
163181

164-
clusters += grouped
182+
clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
183+
return clusters
184+
}
185+
186+
private fun addClustersOfFailedExecutions(
187+
grouped: Map<ExecutionGroup, List<UtExecution>>,
188+
commentPrefix: String,
189+
commentPostfix: String
190+
): List<FailedExecutionCluster> {
191+
val clusters = grouped
165192
.filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
166193
.map { (suffixId, group) ->
167-
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
168-
}
194+
FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
195+
}
196+
169197
return clusters
170198
}
171199

@@ -197,18 +225,18 @@ private fun UtExecutionResult.clusterKind() = when (this) {
197225
/**
198226
* Structure used to represent execution cluster with header
199227
*/
200-
sealed class ExecutionCluster(var header: String, val executions: List<UtSymbolicExecution>)
228+
sealed class ExecutionCluster(var header: String, val executions: List<UtExecution>)
201229

202230
/**
203231
* Represents successful execution cluster
204232
*/
205-
private class SuccessfulExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
233+
private class SuccessfulExecutionCluster(header: String, executions: List<UtExecution>) :
206234
ExecutionCluster(header, executions)
207235

208236
/**
209237
* Represents failed execution cluster
210238
*/
211-
private class FailedExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
239+
private class FailedExecutionCluster(header: String, executions: List<UtExecution>) :
212240
ExecutionCluster(header, executions)
213241

214242
/**

‎utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,27 @@ package org.utbot.summary
22

33
object UtSummarySettings {
44
/**
5-
* If True test comments will be generated
5+
* If True test comments will be generated.
66
*/
77
var GENERATE_COMMENTS = true
88

99
/**
10-
* If True cluster comments will be generated
10+
* If True cluster comments will be generated.
1111
*/
1212
var GENERATE_CLUSTER_COMMENTS = true
1313

1414
/**
15-
* If True names for tests will be generated
15+
* If True names for tests will be generated.
1616
*/
1717
var GENERATE_NAMES = true
1818

1919
/**
20-
* If True display names for tests will be generated
20+
* If True display names for tests will be generated.
2121
*/
2222
var GENERATE_DISPLAY_NAMES = true
2323

2424
/**
25-
* generate display name in from -> to style
25+
* If True display name in from -> to style will be generated.
2626
*/
2727
var GENERATE_DISPLAYNAME_FROM_TO_STYLE = true
2828

@@ -34,19 +34,21 @@ object UtSummarySettings {
3434

3535
/**
3636
* Sets minimum number of successful execution
37-
* for applying the clustering algorithm
37+
* for applying the clustering algorithm.
3838
*/
3939
const val MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING: Int = 4
4040

4141
/**
42-
* DBSCAN hyperparameter
43-
* Sets minimum number of executions to form a cluster
42+
* DBSCAN hyperparameter.
43+
*
44+
* Sets minimum number of executions to form a cluster.
4445
*/
4546
var MIN_EXEC_DBSCAN: Int = 2
4647

4748
/**
48-
* DBSCAN hyperparameter
49-
* Sets radius of search for algorithm
49+
* DBSCAN hyperparameter.
50+
*
51+
* Sets radius of search for algorithm.
5052
*/
5153
var RADIUS_DBSCAN: Float = 5.0f
5254
}

0 commit comments

Comments
 (0)