[JUnit Platform Engine] Improve Maven and Gradle compatibility

Maven Surefire and Gradle do not (yet?) fully support the JUnit
Platform API[1]. To work around this problem the `cucumber.features`
can be used.

However, because Maven Surefire and Gradle only attempt to discover
class based tests, they do expect a class source. As a result of this
missing source, Maven Surefire will not report on the executed tests
while Gradle reports the tests as having been executed by an
"Unknown Class".

By having the Cucumber TestEngine pretend to have a ClassSource when
`cucumber.features` is used both these problems go away.

1. #2498

Author: mpkorstanje

CHANGELOG.md

@@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- [JUnit Platform Engine] Improve Maven and Gradle compatibility ([#2832](https://github.com/cucumber/cucumber-jvm/pull/2832) M.P. Korstanje)
 
 ## [7.15.0] - 2023-12-11
 ### Changed

cucumber-junit-platform-engine/README.md

@@ -160,9 +160,6 @@ To select the scenario on line 10 of the `example.feature` file use:
 mvn test -Dsurefire.includeJUnit5Engines=cucumber -Dcucumber.plugin=pretty -Dcucumber.features=path/to/example.feature:10 
 ```
 
-Note: Add `-Dcucumber.plugin=pretty` to get test reports. Maven will not
-report on tests without a class.
-
 #### Gradle
 
 TODO: (Feel free to send a pull request. ;))
@@ -342,7 +339,13 @@ cucumber.filter.name=                                          # a regular expre
 cucumber.features=                                             # comma separated paths to feature files. 
                                                                # example: path/to/example.feature, path/to/other.feature
                                                                # note: When used any discovery selectors from the JUnit
-                                                               # Platform will be ignored. Use with caution and care.
+                                                               # Platform will be ignored. This may lead to multiple
+                                                               # executions of Cucumber. For example when used in
+                                                               # combination with the JUnit Platform Suite Engine.
+                                                               # When using cucumber through the JUnit Platform
+                                                               # Launcher API or the JUnit Platform Suite Engine, it is
+                                                               # recommended to respectively use JUnits
+                                                               # DiscoverySelectors or equivalent annotations.
 
 cucumber.filter.tags=                                          # a cucumber tag expression.
                                                                # only scenarios with matching tags are executed.

cucumber-junit-platform-engine/src/main/java/io/cucumber/junit/platform/engine/Constants.java

@@ -61,8 +61,19 @@ public final class Constants {
      * scenario or example at line 42 in the example feature file</li>
      * </ul>
      * <p>
-     * NOTE: When used any discovery selectors from the JUnit Platform will be
-     * ignored. Use with caution and care.
+     * Note: When used any discovery selectors from the JUnit Platform will be
+     * ignored. This may lead to multiple executions of Cucumber. For example
+     * when used in combination with the JUnit Platform Suite Engine.
+     * <p>
+     * When using cucumber through the JUnit Platform Launcher API or the JUnit
+     * Platform Suite Engine, it is recommended to respectively use the
+     * {@link org.junit.platform.engine.discovery.DiscoverySelectors} or
+     * equivalent annotations.
+     * <p>
+     * Additionally, when this property is used, to work around limitations in
+     * Maven Surefire and Gradle, the Cucumber Engine will report its
+     * {@link org.junit.platform.engine.TestSource} as
+     * {@link CucumberTestEngine}.
      *
      * @see io.cucumber.core.feature.FeatureWithLines
      */

cucumber-junit-platform-engine/src/main/java/io/cucumber/junit/platform/engine/CucumberEngineDescriptor.java

@@ -1,6 +1,7 @@
 package io.cucumber.junit.platform.engine;
 
 import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestSource;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.support.descriptor.EngineDescriptor;
 import org.junit.platform.engine.support.hierarchical.Node;
@@ -11,9 +12,20 @@
 class CucumberEngineDescriptor extends EngineDescriptor implements Node<CucumberEngineExecutionContext> {
 
     static final String ENGINE_ID = "cucumber";
+    private final TestSource source;
 
     CucumberEngineDescriptor(UniqueId uniqueId) {
+        this(uniqueId, null);
+    }
+
+    CucumberEngineDescriptor(UniqueId uniqueId, TestSource source) {
         super(uniqueId, "Cucumber");
+        this.source = source;
+    }
+
+    @Override
+    public Optional<TestSource> getSource() {
+        return Optional.ofNullable(this.source);
     }
 
     @Override

cucumber-junit-platform-engine/src/main/java/io/cucumber/junit/platform/engine/CucumberTestEngine.java

@@ -5,12 +5,15 @@
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.ExecutionRequest;
 import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestSource;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.support.config.PrefixedConfigurationParameters;
+import org.junit.platform.engine.support.descriptor.ClassSource;
 import org.junit.platform.engine.support.hierarchical.ForkJoinPoolHierarchicalTestExecutorService;
 import org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine;
 import org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutorService;
 
+import static io.cucumber.junit.platform.engine.Constants.FEATURES_PROPERTY_NAME;
 import static io.cucumber.junit.platform.engine.Constants.PARALLEL_CONFIG_PREFIX;
 import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME;
 
@@ -39,11 +42,24 @@ public String getId() {
 
     @Override
     public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
-        CucumberEngineDescriptor engineDescriptor = new CucumberEngineDescriptor(uniqueId);
+        TestSource testSource = createEngineTestSource(discoveryRequest);
+        CucumberEngineDescriptor engineDescriptor = new CucumberEngineDescriptor(uniqueId, testSource);
         new DiscoverySelectorResolver().resolveSelectors(discoveryRequest, engineDescriptor);
         return engineDescriptor;
     }
 
+    private static TestSource createEngineTestSource(EngineDiscoveryRequest discoveryRequest) {
+        // Workaround. Test Engines do not normally have test source.
+        // Maven does not count tests that do not have a ClassSource somewhere
+        // in the test descriptor tree.
+        // Gradle will report all tests as coming from an "Unknown Class"
+        ConfigurationParameters configuration = discoveryRequest.getConfigurationParameters();
+        if (configuration.get(FEATURES_PROPERTY_NAME).isPresent()) {
+            return ClassSource.from(CucumberTestEngine.class);
+        }
+        return null;
+    }
+
     @Override
     protected HierarchicalTestExecutorService createExecutorService(ExecutionRequest request) {
         ConfigurationParameters config = request.getConfigurationParameters();

cucumber-junit-platform-engine/src/test/java/io/cucumber/junit/platform/engine/CucumberTestEngineTest.java

@@ -1,16 +1,23 @@
 package io.cucumber.junit.platform.engine;
 
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.Condition;
 import org.junit.jupiter.api.Test;
 import org.junit.platform.engine.ConfigurationParameters;
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.EngineExecutionListener;
 import org.junit.platform.engine.ExecutionRequest;
 import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestSource;
 import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.support.descriptor.ClassSource;
 import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.platform.testkit.engine.Event;
+import org.junit.platform.testkit.engine.EventConditions;
 
 import java.util.Optional;
 
+import static io.cucumber.junit.platform.engine.Constants.FEATURES_PROPERTY_NAME;
 import static io.cucumber.junit.platform.engine.Constants.FILTER_NAME_PROPERTY_NAME;
 import static io.cucumber.junit.platform.engine.Constants.FILTER_TAGS_PROPERTY_NAME;
 import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PUBLISH_QUIET_PROPERTY_NAME;
@@ -70,6 +77,31 @@ void selectAndExecuteSingleScenario() {
                 .haveExactly(1, event(finishedSuccessfully()));
     }
 
+    @Test
+    void selectAndExecuteSingleScenarioThroughFeaturesProperty() {
+        EngineTestKit.engine(ENGINE_ID)
+                .configurationParameter(PLUGIN_PUBLISH_QUIET_PROPERTY_NAME, "true")
+                .configurationParameter(FEATURES_PROPERTY_NAME,
+                    "src/test/resources/io/cucumber/junit/platform/engine/single.feature")
+                .execute()
+                .allEvents()
+                .assertThatEvents()
+                .haveExactly(2, event(engine(source(ClassSource.from(CucumberTestEngine.class)))))
+                .haveExactly(1, event(test(finishedSuccessfully())));
+    }
+
+    @Test
+    void selectAndExecuteSingleScenarioWithoutFeaturesProperty() {
+        EngineTestKit.engine(ENGINE_ID)
+                .configurationParameter(PLUGIN_PUBLISH_QUIET_PROPERTY_NAME, "true")
+                .selectors(selectFile("src/test/resources/io/cucumber/junit/platform/engine/single.feature"))
+                .execute()
+                .allEvents()
+                .assertThatEvents()
+                .haveExactly(2, event(engine(emptySource())))
+                .haveExactly(1, event(test(finishedSuccessfully())));
+    }
+
     @Test
     void selectAndSkipDisabledScenarioByTags() {
         EngineTestKit.engine(ENGINE_ID)
@@ -98,4 +130,17 @@ void selectAndSkipDisabledScenarioByName() {
                     event(skippedWithReason("'cucumber.filter.name=^Nothing$' did not match this scenario")));
     }
 
+    private static Condition<Event> engine(Condition<Event> condition) {
+        return Assertions.allOf(EventConditions.engine(), condition);
+    }
+
+    private static Condition<Event> source(TestSource testSource) {
+        return new Condition<>(event -> event.getTestDescriptor().getSource().filter(testSource::equals).isPresent(),
+            "test engine with test source '%s'", testSource);
+    }
+
+    private static Condition<Event> emptySource() {
+        return new Condition<>(event -> !event.getTestDescriptor().getSource().isPresent(), "without a test source");
+    }
+
 }