Skip to content

Add tests for fuzzing platform #1516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 15, 2022
Merged

Add tests for fuzzing platform #1516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
009438d
Add smoke tests for fuzzer platform
Markoutte Dec 13, 2022
7f556d5
Add tests for ValueProvider
Markoutte Dec 14, 2022
8b7ea00
Test that random generates same values by default for every run
Markoutte Dec 14, 2022
6b09a94
Result of fuzzing should be reproducible
Markoutte Dec 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,6 @@ class UtBotSymbolicEngine(
methodUnderTest,
collectConstantsForFuzzer(graph),
names,
{ mockStrategy.eligibleToMock(it, classUnderTest) },
listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator))))
) { thisInstance, descr, values ->
if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {
Expand Down
3 changes: 1 addition & 2 deletions utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ suspend fun runJavaFuzzing(
methodUnderTest: ExecutableId,
constants: Collection<FuzzedConcreteValue>,
names: List<String>,
mock: (ClassId) -> Boolean = { false },
providers: List<ValueProvider<FuzzedType, FuzzedValue, FuzzedDescription>> = defaultValueProviders(idGenerator),
exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List<FuzzedValue>) -> BaseFeedback<Trie.Node<Instruction>, FuzzedType, FuzzedValue>
) {
Expand Down Expand Up @@ -83,7 +82,7 @@ suspend fun runJavaFuzzing(
null
}
}
shouldMock = mock
shouldMock = { false }
}

val thisInstance = with(methodUnderTest) {
Expand Down
8 changes: 8 additions & 0 deletions utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package org.utbot.fuzzing.samples;

@SuppressWarnings("unused")
public class Stubs {

public static void name(String value) {}

}
53 changes: 53 additions & 0 deletions utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package org.utbot.fuzzing

import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
import org.utbot.framework.plugin.api.MethodId
import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator
import org.utbot.framework.plugin.api.UtPrimitiveModel
import org.utbot.framework.plugin.api.util.*
import org.utbot.fuzzer.FuzzedConcreteValue
import org.utbot.fuzzing.samples.Stubs
import org.utbot.fuzzing.utils.Trie

class JavaFuzzingTest {

@Test
fun `string generates same values`() {
fun collect(): List<String> {
return runBlockingWithContext {
Comment on lines +16 to +19
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just thoughts: if we want reproducibility in many/most/all cases, maybe we should find a more general way to check it? E.g., introduce something like

fun <T> getReproducibleResults(generator: () -> T): T {
    val probe1 = generator()
    val probe2 = generator()
    assertEquals(probe1, probe2)
    return probe1
}

Then we can wrap calls to fuzzer in tests in such function and process results as usual. (Just an example, there are lots of other ways to check reproducibility more general).

val results = mutableListOf<String>()
var count = 0
val probes = 10000
runJavaFuzzing(
TestIdentityPreservingIdGenerator,
methodUnderTest = MethodId(Stubs::class.id, "name", voidClassId, listOf(stringClassId)),
constants = listOf(
FuzzedConcreteValue(stringClassId, "Hello"),
FuzzedConcreteValue(stringClassId, "World"),
FuzzedConcreteValue(stringClassId, "!"),
),
names = emptyList()
) { _, _, values ->
results += (values.first().model as UtPrimitiveModel).value as String
BaseFeedback(Trie.emptyNode(), if (++count < probes) Control.CONTINUE else Control.STOP)
}
Assertions.assertEquals(count, results.size)
results
}
}

val probe1 = collect()
val probe2 = collect()
Assertions.assertEquals(probe1, probe2)
}

private fun <T> runBlockingWithContext(block: suspend () -> T) : T {
return withUtContext(UtContext(this::class.java.classLoader)) {
runBlocking {
block()
}
}
}
}
15 changes: 6 additions & 9 deletions utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
@file:JvmName("FuzzingApi")
package org.utbot.fuzzing

import kotlinx.coroutines.yield
import mu.KotlinLogging
import org.utbot.fuzzing.seeds.KnownValue
import org.utbot.fuzzing.utils.chooseOne
Expand Down Expand Up @@ -259,7 +260,7 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
val seeds = Statistics<T, R, F>()
run breaking@ {
sequence {
while (true) {
while (description.parameters.isNotEmpty()) {
if (dynamicallyGenerated.isNotEmpty()) {
yield(dynamicallyGenerated.removeFirst())
} else {
Expand All @@ -280,15 +281,11 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
}
}
}.forEach execution@ { values ->
yield()
check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
val valuesCache = mutableMapOf<Result<T, R>, R>()
val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
val feedback = try {
fuzzing.handle(description, result)
} catch (t: Throwable) {
logger.error(t) { "Error when running fuzzing with $values" }
return@execution
}
val feedback = fuzzing.handle(description, result)
when (feedback.control) {
Control.CONTINUE -> {
seeds.put(random, configuration, feedback, values)
Expand Down Expand Up @@ -634,8 +631,8 @@ private class Node<TYPE, RESULT>(


private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
private val seeds = hashMapOf<FEEDBACK, Node<TYPE, RESULT>>()
private val count = hashMapOf<FEEDBACK, Long>()
private val seeds = linkedMapOf<FEEDBACK, Node<TYPE, RESULT>>()
private val count = linkedMapOf<FEEDBACK, Long>()

fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node<TYPE, RESULT>) {
if (random.flipCoin(configuration.probUpdateSeedInsteadOfKeepOld)) {
Expand Down
21 changes: 6 additions & 15 deletions utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ fun interface ValueProvider<T, R, D : Description<T>> {
*
* This model provider is called before `anotherValueProviders`.
*/
fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
infix fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
fun toList(m: ValueProvider<T, R, D>) = if (m is Combined<T, R, D>) m.providers else listOf(m)
return Combined(toList(this) + toList(anotherValueProvider))
}
Expand Down Expand Up @@ -122,15 +122,8 @@ fun interface ValueProvider<T, R, D : Description<T>> {
* Creates new value provider that creates default value if no values are generated by this provider.
*/
fun withFallback(fallbackSupplier: (T) -> Seed<T, R>) : ValueProvider<T, R, D> {
val thisProvider = this
return ValueProvider { description, type ->
if (accept(type)) {
thisProvider.generate(description, type)
.takeIf { it.iterator().hasNext() }
?: sequenceOf(fallbackSupplier(type))
} else {
emptySequence()
}
return withFallback { _, type ->
sequenceOf(fallbackSupplier(type))
}
}

Expand Down Expand Up @@ -170,10 +163,6 @@ fun interface ValueProvider<T, R, D : Description<T>> {
}
}

fun<T, R, D : Description<T>> List<ValueProvider<T, R, D>>.withFallback(fallbackSupplier: (T) -> Seed<T, R>) : List<ValueProvider<T, R, D>> {
return listOf(ValueProvider.of(this).withFallback(fallbackSupplier))
}

/**
* Simple value provider for a concrete type.
*
Expand All @@ -186,6 +175,8 @@ class TypeProvider<T, R, D : Description<T>>(
) : ValueProvider<T, R, D> {
override fun accept(type: T) = this.type == type
override fun generate(description: D, type: T) = sequence {
this.generate(description, type)
if (accept(type)) {
this.generate(description, type)
}
}
}
3 changes: 2 additions & 1 deletion utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ private class JsonBuilder(
*/
@Suppress("RemoveExplicitTypeArguments")
suspend fun main() {
var count = 0
BaseFuzzing<CustomType, JsonBuilder, Description<CustomType>, Feedback<CustomType, JsonBuilder>>(
TypeProvider(CustomType.INT) { _, _ ->
for (b in Signed.values()) {
Expand Down Expand Up @@ -79,7 +80,7 @@ suspend fun main() {
},
) { _, values ->
println(values)
emptyFeedback()
if (++count < 1000) emptyFeedback() else error("")
}.fuzz(
Description(listOf(CustomType.LST, CustomType.OBJ)),
Random(0),
Expand Down
6 changes: 0 additions & 6 deletions utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,6 @@ open class Trie<T, K>(
get() = error("empty node has no data")
override val count: Int
get() = 0
override fun equals(other: Any?): Boolean {
return false
}
override fun hashCode(): Int {
return 0
}
}

companion object {
Expand Down
Loading