Configurable lifecycle mode

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -69,13 +69,34 @@ java_fuzz_target_test(
 )
 
 java_fuzz_target_test(
-    name = "LifecycleFuzzTest",
-    srcs = ["LifecycleFuzzTest.java"],
+    name = "PerExecutionLifecycleFuzzTest",
+    srcs = ["PerExecutionLifecycleFuzzTest.java"],
     allowed_findings = ["com.example.TestSuccessfulException"],
     fuzzer_args = [
         "-runs=3",
     ],
-    target_class = "com.example.LifecycleFuzzTest",
+    target_class = "com.example.PerExecutionLifecycleFuzzTest",
+    verify_crash_reproducer = False,
+    runtime_deps = [
+        ":junit_runtime",
+    ],
+    deps = [
+        ":test_successful_exception",
+        "//examples/junit/src/main/java/com/example:parser",
+        "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:com_google_truth_truth",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
+    ],
+)
+
+java_fuzz_target_test(
+    name = "PerTestLifecycleFuzzTest",
+    srcs = ["PerTestLifecycleFuzzTest.java"],
+    allowed_findings = ["com.example.TestSuccessfulException"],
+    fuzzer_args = [
+        "-runs=3",
+    ],
+    target_class = "com.example.PerTestLifecycleFuzzTest",
     verify_crash_reproducer = False,
     runtime_deps = [
         ":junit_runtime",

examples/junit/src/test/java/com/example/PerExecutionLifecycleFuzzTest.java

@@ -0,0 +1,197 @@
+/*
+ * Copyright 2022 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+
+import com.code_intelligence.jazzer.junit.FuzzTest;
+import com.code_intelligence.jazzer.junit.Lifecycle;
+import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks1;
+import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks2;
+import com.example.PerExecutionLifecycleFuzzTest.LifecycleCallbacks3;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestInstancePostProcessor;
+
+@TestMethodOrder(MethodOrderer.MethodName.class)
+@ExtendWith(PerExecutionLifecycleFuzzTest.LifecycleInstancePostProcessor.class)
+@ExtendWith(LifecycleCallbacks1.class)
+@ExtendWith(LifecycleCallbacks2.class)
+@ExtendWith(LifecycleCallbacks3.class)
+class PerExecutionLifecycleFuzzTest {
+  private static final ArrayList<String> events = new ArrayList<>();
+  private static final long RUNS = 3;
+
+  private boolean beforeEachCalledOnInstance = false;
+  private boolean testInstancePostProcessorCalledOnInstance = false;
+
+  @BeforeAll
+  static void beforeAll() {
+    events.add("beforeAll");
+  }
+
+  @BeforeEach
+  void beforeEach1() {
+    events.add("beforeEach1");
+    beforeEachCalledOnInstance = true;
+  }
+
+  @BeforeEach
+  void beforeEach2() {
+    events.add("beforeEach2");
+  }
+
+  @BeforeEach
+  void beforeEach3() {
+    events.add("beforeEach3");
+  }
+
+  @Disabled
+  @FuzzTest
+  void disabledFuzz(byte[] data) {
+    events.add("disabledFuzz");
+    throw new AssertionError("This test should not be executed");
+  }
+
+  @FuzzTest(maxExecutions = RUNS, lifecycle = Lifecycle.PER_EXECUTION)
+  void lifecycleFuzz(byte[] data) {
+    events.add("lifecycleFuzz");
+    assertThat(beforeEachCalledOnInstance).isTrue();
+    assertThat(testInstancePostProcessorCalledOnInstance).isTrue();
+  }
+
+  @AfterEach
+  void afterEach1() {
+    events.add("afterEach1");
+  }
+
+  @AfterEach
+  void afterEach2() {
+    events.add("afterEach2");
+  }
+
+  @AfterEach
+  void afterEach3() {
+    events.add("afterEach3");
+  }
+
+  @AfterAll
+  static void afterAll() throws TestSuccessfulException {
+    events.add("afterAll");
+
+    boolean isRegressionTest = "".equals(System.getenv("JAZZER_FUZZ"));
+    boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null;
+    boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest;
+
+    final List<String> expectedBeforeEachEvents =
+        unmodifiableList(
+            asList(
+                "beforeEachCallback1",
+                "beforeEachCallback2",
+                "beforeEachCallback3",
+                "beforeEach1",
+                "beforeEach2",
+                "beforeEach3"));
+    final List<String> expectedAfterEachEvents =
+        unmodifiableList(
+            asList(
+                "afterEach1",
+                "afterEach2",
+                "afterEach3",
+                "afterEachCallback3",
+                "afterEachCallback2",
+                "afterEachCallback1"));
+
+    ArrayList<String> expectedEvents = new ArrayList<>();
+    expectedEvents.add("beforeAll");
+
+    // When run from the command-line, the fuzz test is not separately executed on the empty seed.
+    if (isRegressionTest || isFuzzingFromJUnit) {
+      expectedEvents.addAll(expectedBeforeEachEvents);
+      expectedEvents.add("lifecycleFuzz");
+      expectedEvents.addAll(expectedAfterEachEvents);
+    }
+    if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
+      for (int i = 0; i < RUNS; i++) {
+        expectedEvents.addAll(expectedBeforeEachEvents);
+        expectedEvents.add("lifecycleFuzz");
+        expectedEvents.addAll(expectedAfterEachEvents);
+      }
+    }
+
+    expectedEvents.add("afterAll");
+
+    assertThat(events).containsExactlyElementsIn(expectedEvents).inOrder();
+    throw new TestSuccessfulException("Lifecycle methods invoked as expected");
+  }
+
+  static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
+    @Override
+    public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
+      ((PerExecutionLifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
+    }
+  }
+
+  static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback1");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback1");
+    }
+  }
+
+  static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback2");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback2");
+    }
+  }
+
+  static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext extensionContext) {
+      events.add("beforeEachCallback3");
+    }
+
+    @Override
+    public void afterEach(ExtensionContext extensionContext) {
+      events.add("afterEachCallback3");
+    }
+  }
+}

examples/junit/src/test/java/com/example/LifecycleFuzzTest.java → examples/junit/src/test/java/com/example/PerTestLifecycleFuzzTest.java

@@ -21,9 +21,9 @@
 import static java.util.Collections.unmodifiableList;
 
 import com.code_intelligence.jazzer.junit.FuzzTest;
-import com.example.LifecycleFuzzTest.LifecycleCallbacks1;
-import com.example.LifecycleFuzzTest.LifecycleCallbacks2;
-import com.example.LifecycleFuzzTest.LifecycleCallbacks3;
+import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks1;
+import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks2;
+import com.example.PerTestLifecycleFuzzTest.LifecycleCallbacks3;
 import java.util.ArrayList;
 import java.util.List;
 import org.junit.jupiter.api.AfterAll;
@@ -40,11 +40,11 @@
 import org.junit.jupiter.api.extension.TestInstancePostProcessor;
 
 @TestMethodOrder(MethodOrderer.MethodName.class)
-@ExtendWith(LifecycleFuzzTest.LifecycleInstancePostProcessor.class)
+@ExtendWith(PerTestLifecycleFuzzTest.LifecycleInstancePostProcessor.class)
 @ExtendWith(LifecycleCallbacks1.class)
 @ExtendWith(LifecycleCallbacks2.class)
 @ExtendWith(LifecycleCallbacks3.class)
-class LifecycleFuzzTest {
+class PerTestLifecycleFuzzTest {
   private static final ArrayList<String> events = new ArrayList<>();
   private static final long RUNS = 3;
 
@@ -138,11 +138,11 @@ static void afterAll() throws TestSuccessfulException {
       expectedEvents.addAll(expectedAfterEachEvents);
     }
     if (isFuzzingFromJUnit || isFuzzingFromCommandLine) {
+      expectedEvents.addAll(expectedBeforeEachEvents);
       for (int i = 0; i < RUNS; i++) {
-        expectedEvents.addAll(expectedBeforeEachEvents);
         expectedEvents.add("lifecycleFuzz");
-        expectedEvents.addAll(expectedAfterEachEvents);
       }
+      expectedEvents.addAll(expectedAfterEachEvents);
     }
 
     expectedEvents.add("afterAll");
@@ -154,7 +154,7 @@ static void afterAll() throws TestSuccessfulException {
   static class LifecycleInstancePostProcessor implements TestInstancePostProcessor {
     @Override
     public void postProcessTestInstance(Object o, ExtensionContext extensionContext) {
-      ((LifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
+      ((PerTestLifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true;
     }
   }
 

src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java

@@ -22,8 +22,7 @@
 import org.junit.jupiter.params.provider.ArgumentsProvider;
 import org.junit.jupiter.params.support.AnnotationConsumer;
 
-public class AgentConfiguringArgumentsProvider
-    implements ArgumentsProvider, AnnotationConsumer<FuzzTest> {
+class AgentConfiguringArgumentsProvider implements ArgumentsProvider, AnnotationConsumer<FuzzTest> {
   private FuzzTest fuzzTest;
 
   @Override

src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel

@@ -33,6 +33,9 @@ java_library(
         "//examples/junit/src/test/java/com/example:__pkg__",
         "//selffuzz/src/test/java/com/code_intelligence/selffuzz:__subpackages__",
     ],
+    exports = [
+        ":lifecycle",
+    ],
     runtime_deps = [
         # The JUnit launcher that is part of the Jazzer driver needs this on the classpath
         # to run an @FuzzTest with JUnit. This will also result in a transitive dependency
@@ -42,6 +45,7 @@ java_library(
     deps = [
         ":fuzz_test_configuration_error",
         ":fuzz_test_executor",
+        ":lifecycle",
         ":seed_serializer",
         ":utils",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
@@ -67,6 +71,7 @@ java_jni_library(
     deps = [
         ":agent_configurator",
         ":junit_lifecycle_methods_invoker",
+        ":lifecycle",
         ":seed_serializer",
         ":utils",
         "//src/main/java/com/code_intelligence/jazzer/agent:agent_installer",
@@ -103,12 +108,18 @@ java_library(
     srcs = ["JUnitLifecycleMethodsInvoker.java"],
     deps = [
         ":junit_internals_compile_only",
+        ":lifecycle",
         "//src/main/java/com/code_intelligence/jazzer/driver:lifecycle_methods_invoker",
         "//src/main/java/com/code_intelligence/jazzer/utils:unsafe_provider",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
 )
 
+java_library(
+    name = "lifecycle",
+    srcs = ["Lifecycle.java"],
+)
+
 java_library(
     name = "seed_serializer",
     srcs = ["SeedSerializer.java"],

src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java

@@ -124,6 +124,14 @@
    * <p>This option has no effect during regression testing.
    */
   long maxExecutions() default 0;
+
+  /**
+   * Controls the JUnit lifecycle of fuzz tests during fuzzing.
+   *
+   * <p>During regression testing, fuzz tests always go through the full JUnit lifecycle for every
+   * execution regardless of the value of this option.
+   */
+  Lifecycle lifecycle() default Lifecycle.PER_TEST;
 }
 
 // Internal use only.

src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java

@@ -312,7 +312,8 @@ public void addSeed(byte[] bytes) throws IOException {
   public Optional<Throwable> execute(
       ReflectiveInvocationContext<Method> invocationContext,
       ExtensionContext extensionContext,
-      SeedSerializer seedSerializer) {
+      SeedSerializer seedSerializer,
+      Lifecycle lifecycle) {
     if (seedSerializer instanceof AutofuzzSeedSerializer) {
       FuzzTargetHolder.fuzzTarget =
           FuzzTargetHolder.autofuzzFuzzTarget(
@@ -336,7 +337,7 @@ public Optional<Throwable> execute(
           new FuzzTargetHolder.FuzzTarget(
               invocationContext.getExecutable(),
               () -> invocationContext.getTarget().get(),
-              JUnitLifecycleMethodsInvoker.of(extensionContext));
+              JUnitLifecycleMethodsInvoker.of(extensionContext, lifecycle));
     }
 
     // Only register a finding handler in case the fuzz test is executed by JUnit.