[Core] Support parallel execution of pickles (cucumber#1389)

Adds supports for parallel execution of pickles to cucumber-jvm

# Added --threads argument to runtime options

Allows users of the CLI to specify the (max) number of threads to be used to run the tests. TestNG/JUnit users should consult their documentation on how to run JUnit/TestNG in parallel.

Note that JUnit will only run features in parallel, not scenarios.

# Concurrent Events

During parallel executing events from the execution of different pickles may interleave. To avoid breaking existing Formatters, these will now receive all test events after the run is complete. Because we are unable to infer whether JUnit/TestNG run in parallel the assumption is that they are and their formatter will always get all events after the run. 

Formatters that can handle concurrent events can events in real time by implementing the ConcurrentEventListener.

# New formatter introduced TimelineFormatter

Which produces reports using vsjis.org timeline to highlight which feature was run on which Thread and when.

Author: boaty82

core/pom.xml

@@ -64,6 +64,18 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>uk.co.datumedge</groupId>
+            <artifactId>hamcrest-json</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

core/src/main/java/cucumber/api/cli/Main.java

@@ -1,34 +1,12 @@
 package cucumber.api.cli;
 
-import cucumber.runner.EventBus;
-import cucumber.runner.TimeService;
-import cucumber.runtime.BackendModuleBackendSupplier;
-import cucumber.runtime.ClassFinder;
-import cucumber.runtime.ExitStatus;
-import cucumber.runtime.FeaturePathFeatureSupplier;
-import cucumber.runtime.FeatureSupplier;
-import cucumber.runtime.GlueSupplier;
-import cucumber.runtime.RunnerSupplier;
-import cucumber.runtime.filter.Filters;
-import cucumber.runtime.formatter.Plugins;
-import cucumber.runtime.filter.RerunFilters;
-import cucumber.runtime.ThreadLocalRunnerSupplier;
-import cucumber.runtime.RuntimeGlueSupplier;
 import cucumber.runtime.Runtime;
-import cucumber.runtime.RuntimeOptions;
-import cucumber.runtime.formatter.PluginFactory;
-import cucumber.runtime.io.MultiLoader;
-import cucumber.runtime.io.ResourceLoader;
-import cucumber.runtime.io.ResourceLoaderClassFinder;
-import cucumber.runtime.model.FeatureLoader;
-
-import static java.util.Arrays.asList;
 
 public class Main {
 
     public static void main(String[] argv) {
-        byte exitstatus = run(argv, Thread.currentThread().getContextClassLoader());
-        System.exit(exitstatus);
+        byte exitStatus = run(argv, Thread.currentThread().getContextClassLoader());
+        System.exit(exitStatus);
     }
 
     /**
@@ -39,23 +17,13 @@ public static void main(String[] argv) {
      * @return 0 if execution was successful, 1 if it was not (test failures)
      */
     public static byte run(String[] argv, ClassLoader classLoader) {
-        RuntimeOptions runtimeOptions = new RuntimeOptions(asList(argv));
 
-        ResourceLoader resourceLoader = new MultiLoader(classLoader);
-        ClassFinder classFinder = new ResourceLoaderClassFinder(resourceLoader, classLoader);
-        BackendModuleBackendSupplier backendSupplier = new BackendModuleBackendSupplier(resourceLoader, classFinder, runtimeOptions);
-        EventBus bus = new EventBus(TimeService.SYSTEM);
-        Plugins plugins = new Plugins(classLoader, new PluginFactory(), bus, runtimeOptions);
-        ExitStatus exitStatus = new ExitStatus(runtimeOptions);
-        exitStatus.setEventPublisher(bus);
-        GlueSupplier glueSupplier = new RuntimeGlueSupplier();
-        RunnerSupplier runnerSupplier = new ThreadLocalRunnerSupplier(runtimeOptions, bus, backendSupplier, glueSupplier);
-        FeatureLoader featureLoader = new FeatureLoader(resourceLoader);
-        FeatureSupplier featureSupplier = new FeaturePathFeatureSupplier(featureLoader, runtimeOptions);
-        RerunFilters rerunFilters = new RerunFilters(runtimeOptions, featureLoader);
-        Filters filters = new Filters(runtimeOptions, rerunFilters);
-        Runtime runtime = new Runtime(plugins, bus, filters, runnerSupplier, featureSupplier);
+        final Runtime runtime = Runtime.builder()
+            .withArgs(argv)
+            .withClassLoader(classLoader)
+            .build();
+
         runtime.run();
-        return exitStatus.exitStatus();
+        return runtime.exitStatus();
     }
 }

core/src/main/java/cucumber/api/event/CanonicalEventOrder.java

@@ -0,0 +1,78 @@
+package cucumber.api.event;
+
+import java.util.Comparator;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+
+final class CanonicalEventOrder implements Comparator<Event> {
+
+    private static final FixedEventOrderComparator fixedOrder = new FixedEventOrderComparator();
+    private static final TestCaseEventComparator testCaseOrder = new TestCaseEventComparator();
+
+    @Override
+    public int compare(Event a, Event b) {
+        int fixedOrder = CanonicalEventOrder.fixedOrder.compare(a, b);
+        if (fixedOrder != 0) {
+            return fixedOrder;
+        }
+
+        if (!(a instanceof TestCaseEvent && b instanceof TestCaseEvent)) {
+            return fixedOrder;
+        }
+
+        return testCaseOrder.compare((TestCaseEvent) a, (TestCaseEvent) b);
+    }
+
+    private static final class FixedEventOrderComparator implements Comparator<Event> {
+
+        private final List<Class<? extends Event>> fixedOrder = asList(
+            (Class<? extends Event>)
+                TestRunStarted.class,
+            TestSourceRead.class,
+            SnippetsSuggestedEvent.class,
+            TestCaseEvent.class,
+            TestRunFinished.class
+        );
+
+        @Override
+        public int compare(final Event a, final Event b) {
+            return Integer.compare(requireInFixOrder(a.getClass()), requireInFixOrder(b.getClass()));
+        }
+
+        private int requireInFixOrder(Class<? extends Event> o) {
+            int index = findInFixedOrder(o);
+            if (index < 0) {
+                throw new IllegalStateException(o + "was not in " + fixedOrder);
+            }
+            return index;
+        }
+
+        private int findInFixedOrder(Class<? extends Event> o) {
+            for (int i = 0; i < fixedOrder.size(); i++) {
+                if (fixedOrder.get(i).isAssignableFrom(o)) {
+                    return i;
+                }
+            }
+            return -1;
+        }
+    }
+
+    private static final class TestCaseEventComparator implements Comparator<TestCaseEvent> {
+
+        @Override
+        public int compare(TestCaseEvent a, TestCaseEvent b) {
+            int uri = a.testCase.getUri().compareTo(b.testCase.getUri());
+            if (uri != 0) {
+                return uri;
+            }
+
+            int line = Integer.compare(a.testCase.getLine(), b.testCase.getLine());
+            if(line != 0){
+                return line;
+            }
+
+            return Long.compare(a.getTimeStamp(), b.getTimeStamp());
+        }
+    }
+}

core/src/main/java/cucumber/api/event/ConcurrentEventListener.java

@@ -0,0 +1,37 @@
+package cucumber.api.event;
+
+/**
+ * When cucumber executes test in parallel or in a framework
+ * that supports parallel execution (e.g. JUnit or TestNG)
+ * {@link cucumber.api.TestCase} events from different
+ * pickles may interleave.
+ * <p>
+ * This interface marks an {@link EventListener} as capable of
+ * understanding interleaved pickle events.
+ * <p>
+ * While running tests in parallel cucumber makes the
+ * following guarantees.
+ * <p>
+ * 1. The event publisher is synchronized. Events are not
+ * handled concurrently.
+ * <p>
+ * 2. For test cases executed on different threads the callbacks
+ * registered on the event publisher will be called by
+ * different threads. I.e.  Thread.currentThread()
+ * will return different a different thread for two test cases
+ * executed on a different thread (but not necessarily the
+ * executing thread).
+ * <p>
+ *
+ * @see Event
+ */
+public interface ConcurrentEventListener {
+
+    /**
+     * Set the event publisher. The formatter can register event listeners with the publisher.
+     *
+     * @param publisher the event publisher
+     */
+    void setEventPublisher(EventPublisher publisher);
+
+}

core/src/main/java/cucumber/api/event/EmbedEvent.java

@@ -1,11 +1,13 @@
 package cucumber.api.event;
 
-public final class EmbedEvent extends TimeStampedEvent {
+import cucumber.api.TestCase;
+
+public final class EmbedEvent extends TestCaseEvent {
     public final byte[] data;
     public final String mimeType;
 
-    public EmbedEvent(Long timeStamp, byte[] data, String mimeType) {
-        super(timeStamp);
+    public EmbedEvent(Long timeStamp, TestCase testCase, byte[] data, String mimeType) {
+        super(timeStamp, testCase);
         this.data = data;
         this.mimeType = mimeType;
     }

core/src/main/java/cucumber/api/event/Event.java

@@ -1,7 +1,35 @@
 package cucumber.api.event;
 
+import java.util.Comparator;
+
 public interface Event {
 
+    /**
+     * When pickles are executed in parallel or random order
+     * events can be produced with a partial ordering.
+     * <p>
+     * The canonical order is the order in which these events
+     * would have been generated had cucumber executed these
+     * pickles is executed in a serial fashion.
+     * <p>
+     * In canonical order events are first ordered by type:
+     * <ol>
+     * <li>TestRunStarted
+     * <li>TestSourceRead
+     * <li>SnippetsSuggestedEvent
+     * <li>TestCaseEvent
+     * <li>TestRunFinished
+     * </ol>
+     * <p>
+     * Then TestCaseEvents are ordered by
+     * <ol>
+     * <li>uri
+     * <li>line
+     * <li>timestamp
+     * </ol>
+     */
+    Comparator<Event> CANONICAL_ORDER = new CanonicalEventOrder();
+
     Long getTimeStamp();
 
 }

core/src/main/java/cucumber/api/event/EventPublisher.java

@@ -4,13 +4,14 @@ public interface EventPublisher {
 
     /**
      * Registers an event handler for a specific event.
-     *
+     * <p>
      * The available events types are:
      * <ul>
+     * <li>{@link Event} - all events.
      * <li>{@link TestRunStarted} - the first event sent.
      * <li>{@link TestSourceRead} - sent for each feature file read, contains the feature file source.
      * <li>{@link SnippetsSuggestedEvent} - sent for each step that could not be matched to a step definition, contains the raw snippets for the step.
-     * <li> {@link TestCaseStarted} - sent before starting the execution of a Test Case(/Pickle/Scenario), contains the Test Case
+     * <li>{@link TestCaseStarted} - sent before starting the execution of a Test Case(/Pickle/Scenario), contains the Test Case
      * <li>{@link TestStepStarted} - sent before starting the execution of a Test Step, contains the Test Step
      * <li>{@link EmbedEvent} - calling scenario.embed in a hook triggers this event.
      * <li>{@link WriteEvent} - calling scenario.write in a hook triggers this event.
@@ -19,10 +20,20 @@ public interface EventPublisher {
      * <li>{@link TestRunFinished} - the last event sent.
      * </ul>
      *
-     *
      * @param eventType the event type for which the handler is being registered
-     * @param handler the event handler
-     * @param <T> the event type
+     * @param handler   the event handler
+     * @param <T>       the event type
+     * @see Event
      */
     <T extends Event> void registerHandlerFor(Class<T> eventType, EventHandler<T> handler);
+
+    /**
+     * Unregister an event handler for a specific event
+     *
+     * @param eventType the event type for which the handler is being registered
+     * @param handler   the event handler
+     * @param <T>       the event type
+     */
+    <T extends Event> void removeHandlerFor(Class<T> eventType, EventHandler<T> handler);
+
 }

core/src/main/java/cucumber/api/event/TestCaseEvent.java

@@ -0,0 +1,17 @@
+package cucumber.api.event;
+
+import cucumber.api.TestCase;
+
+public abstract class TestCaseEvent extends TimeStampedEvent {
+
+    final TestCase testCase;
+
+    TestCaseEvent(Long timeStamp, TestCase testCase) {
+        super(timeStamp);
+        this.testCase = testCase;
+    }
+
+    public TestCase getTestCase() {
+        return testCase;
+    }
+}

core/src/main/java/cucumber/api/event/TestCaseFinished.java

@@ -3,12 +3,12 @@
 import cucumber.api.Result;
 import cucumber.api.TestCase;
 
-public final class TestCaseFinished extends TimeStampedEvent {
+public final class TestCaseFinished extends TestCaseEvent {
     public final Result result;
     public final TestCase testCase;
 
     public TestCaseFinished(Long timeStamp, TestCase testCase, Result result) {
-        super(timeStamp);
+        super(timeStamp, testCase);
         this.testCase = testCase;
         this.result = result;
     }

core/src/main/java/cucumber/api/event/TestCaseStarted.java

@@ -2,11 +2,11 @@
 
 import cucumber.api.TestCase;
 
-public final class TestCaseStarted extends TimeStampedEvent {
+public final class TestCaseStarted extends TestCaseEvent {
     public final TestCase testCase;
 
     public TestCaseStarted(Long timeStamp, TestCase testCase) {
-        super(timeStamp);
+        super(timeStamp, testCase);
         this.testCase = testCase;
     }
 

core/src/main/java/cucumber/api/event/TestStepFinished.java

@@ -3,6 +3,7 @@
 import cucumber.api.HookTestStep;
 import cucumber.api.PickleStepTestStep;
 import cucumber.api.Result;
+import cucumber.api.TestCase;
 import cucumber.api.TestStep;
 
 /**
@@ -21,12 +22,12 @@
  * @see PickleStepTestStep
  * @see HookTestStep
  */
-public final class TestStepFinished extends TimeStampedEvent {
+public final class TestStepFinished extends TestCaseEvent {
     public final TestStep testStep;
     public final Result result;
 
-    public TestStepFinished(Long timeStamp, TestStep testStep, Result result) {
-        super(timeStamp);
+    public TestStepFinished(Long timeStamp, TestCase testCase, TestStep testStep, Result result) {
+        super(timeStamp, testCase);
         this.testStep = testStep;
         this.result = result;
     }