Add Ordering, Orderable and @OrderWith (#1130)

* Add Ordering, Orderable and @OrderWith.

These APIs allow arbitrary ordering of tests, including randomization.

Author: kcooney

src/main/java/junit/framework/JUnit4TestAdapter.java

@@ -9,8 +9,10 @@
 import org.junit.runner.Runner;
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.manipulation.Filterable;
+import org.junit.runner.manipulation.Orderer;
+import org.junit.runner.manipulation.InvalidOrderingException;
 import org.junit.runner.manipulation.NoTestsRemainException;
-import org.junit.runner.manipulation.Sortable;
+import org.junit.runner.manipulation.Orderable;
 import org.junit.runner.manipulation.Sorter;
 
 /**
@@ -23,7 +25,7 @@ public static Test suite() {
       }
 </pre>
  */
-public class JUnit4TestAdapter implements Test, Filterable, Sortable, Describable {
+public class JUnit4TestAdapter implements Test, Filterable, Orderable, Describable {
     private final Class<?> fNewTestClass;
 
     private final Runner fRunner;
@@ -93,4 +95,13 @@ public void filter(Filter filter) throws NoTestsRemainException {
     public void sort(Sorter sorter) {
         sorter.apply(fRunner);
     }
+
+    /**
+     * {@inheritDoc}
+     *
+     * @since 4.13
+     */
+    public void order(Orderer orderer) throws InvalidOrderingException {
+        orderer.apply(fRunner);
+    }
 }

src/main/java/org/junit/internal/requests/ClassRequest.java

@@ -1,25 +1,18 @@
 package org.junit.internal.requests;
 
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
 import org.junit.internal.builders.AllDefaultPossibilitiesBuilder;
 import org.junit.internal.builders.SuiteMethodBuilder;
-import org.junit.runner.Request;
 import org.junit.runner.Runner;
 import org.junit.runners.model.RunnerBuilder;
 
-public class ClassRequest extends Request {
-    private final Lock runnerLock = new ReentrantLock();
-
+public class ClassRequest extends MemoizingRequest {
     /*
      * We have to use the f prefix, because IntelliJ's JUnit4IdeaTestRunner uses
      * reflection to access this field. See
      * https://github.com/junit-team/junit4/issues/960
      */
     private final Class<?> fTestClass;
     private final boolean canUseSuiteMethod;
-    private volatile Runner runner;
 
     public ClassRequest(Class<?> testClass, boolean canUseSuiteMethod) {
         this.fTestClass = testClass;
@@ -31,18 +24,8 @@ public ClassRequest(Class<?> testClass) {
     }
 
     @Override
-    public Runner getRunner() {
-        if (runner == null) {
-            runnerLock.lock();
-            try {
-                if (runner == null) {
-                    runner = new CustomAllDefaultPossibilitiesBuilder().safeRunnerForClass(fTestClass);
-                }
-            } finally {
-                runnerLock.unlock();
-            }
-        }
-        return runner;
+    protected Runner createRunner() {
+        return new CustomAllDefaultPossibilitiesBuilder().safeRunnerForClass(fTestClass);
     }
 
     private class CustomAllDefaultPossibilitiesBuilder extends AllDefaultPossibilitiesBuilder {

src/main/java/org/junit/internal/requests/MemoizingRequest.java

@@ -0,0 +1,30 @@
+package org.junit.internal.requests;
+
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.junit.runner.Request;
+import org.junit.runner.Runner;
+
+abstract class MemoizingRequest extends Request {
+    private final Lock runnerLock = new ReentrantLock();
+    private volatile Runner runner;
+
+    @Override
+    public final Runner getRunner() {
+        if (runner == null) {
+            runnerLock.lock();
+            try {
+                if (runner == null) {
+                    runner = createRunner();
+                }
+            } finally {
+                runnerLock.unlock();
+            }
+        }
+        return runner;
+    }
+
+    /** Creates the {@link Runner} to return from {@link #getRunner()}. Called at most once. */
+    protected abstract Runner createRunner();
+}

src/main/java/org/junit/internal/requests/OrderingRequest.java

@@ -0,0 +1,29 @@
+package org.junit.internal.requests;
+
+import org.junit.internal.runners.ErrorReportingRunner;
+import org.junit.runner.Request;
+import org.junit.runner.Runner;
+import org.junit.runner.manipulation.InvalidOrderingException;
+import org.junit.runner.manipulation.Ordering;
+
+/** @since 4.13 */
+public class OrderingRequest extends MemoizingRequest {
+    private final Request request;
+    private final Ordering ordering;
+
+    public OrderingRequest(Request request, Ordering ordering) {
+        this.request = request;
+        this.ordering = ordering;
+    }
+
+    @Override
+    protected Runner createRunner() {
+        Runner runner = request.getRunner();
+        try {
+            ordering.apply(runner);
+        } catch (InvalidOrderingException e) {
+            return new ErrorReportingRunner(ordering.getClass(), e);
+        }
+        return runner;
+    }
+}

src/main/java/org/junit/internal/runners/JUnit38ClassRunner.java

@@ -1,5 +1,8 @@
 package org.junit.internal.runners;
 
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+
 import junit.extensions.TestDecorator;
 import junit.framework.AssertionFailedError;
 import junit.framework.Test;
@@ -12,15 +15,16 @@
 import org.junit.runner.Runner;
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.manipulation.Filterable;
+import org.junit.runner.manipulation.Orderer;
+import org.junit.runner.manipulation.InvalidOrderingException;
 import org.junit.runner.manipulation.NoTestsRemainException;
+import org.junit.runner.manipulation.Orderable;
 import org.junit.runner.manipulation.Sortable;
 import org.junit.runner.manipulation.Sorter;
 import org.junit.runner.notification.Failure;
 import org.junit.runner.notification.RunNotifier;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Method;
 
-public class JUnit38ClassRunner extends Runner implements Filterable, Sortable {
+public class JUnit38ClassRunner extends Runner implements Filterable, Orderable {
     private static final class OldTestClassAdaptingListener implements
             TestListener {
         private final RunNotifier notifier;
@@ -170,6 +174,18 @@ public void sort(Sorter sorter) {
         }
     }
 
+    /**
+     * {@inheritDoc}
+     *
+     * @since 4.13
+     */
+    public void order(Orderer orderer) throws InvalidOrderingException {
+        if (getTest() instanceof Orderable) {
+            Orderable adapter = (Orderable) getTest();
+            adapter.order(orderer);
+        }
+    }
+
     private void setTest(Test test) {
         this.test = test;
     }

src/main/java/org/junit/runner/OrderWith.java

@@ -0,0 +1,26 @@
+package org.junit.runner;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.runner.manipulation.Ordering;
+
+/**
+ * When a test class is annotated with <code>&#064;OrderWith</code> or extends a class annotated
+ * with <code>&#064;OrderWith</code>, JUnit will order the tests in the test class (and child
+ * test classes, if any) using the ordering defined by the {@link Ordering} class.
+ *
+ * @since 4.13
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+@Inherited
+public @interface OrderWith {
+    /**
+     * Gets a class that extends {@link Ordering}. The class must have a public no-arg constructor.
+     */
+    Class<? extends Ordering.Factory> value();
+}

src/main/java/org/junit/runner/Request.java

@@ -5,9 +5,11 @@
 import org.junit.internal.builders.AllDefaultPossibilitiesBuilder;
 import org.junit.internal.requests.ClassRequest;
 import org.junit.internal.requests.FilterRequest;
+import org.junit.internal.requests.OrderingRequest;
 import org.junit.internal.requests.SortingRequest;
 import org.junit.internal.runners.ErrorReportingRunner;
 import org.junit.runner.manipulation.Filter;
+import org.junit.runner.manipulation.Ordering;
 import org.junit.runners.model.InitializationError;
 
 /**
@@ -151,15 +153,15 @@ public Request filterWith(Description desiredDescription) {
      * For example, here is code to run a test suite in alphabetical order:
      * <pre>
      * private static Comparator&lt;Description&gt; forward() {
-     * return new Comparator&lt;Description&gt;() {
-     * public int compare(Description o1, Description o2) {
-     * return o1.getDisplayName().compareTo(o2.getDisplayName());
-     * }
-     * };
+     *   return new Comparator&lt;Description&gt;() {
+     *     public int compare(Description o1, Description o2) {
+     *       return o1.getDisplayName().compareTo(o2.getDisplayName());
+     *     }
+     *   };
      * }
      *
      * public static main() {
-     * new JUnitCore().run(Request.aClass(AllTests.class).sortWith(forward()));
+     *   new JUnitCore().run(Request.aClass(AllTests.class).sortWith(forward()));
      * }
      * </pre>
      *
@@ -169,4 +171,32 @@ public Request filterWith(Description desiredDescription) {
     public Request sortWith(Comparator<Description> comparator) {
         return new SortingRequest(this, comparator);
     }
+
+    /**
+     * Returns a Request whose Tests can be run in a certain order, defined by
+     * <code>ordering</code>
+     * <p>
+     * For example, here is code to run a test suite in reverse order:
+     * <pre>
+     * private static Ordering reverse() {
+     *   return new Ordering() {
+     *     public List&lt;Description&gt; orderItems(Collection&lt;Description&gt; descriptions) {
+     *       List&lt;Description&gt; ordered = new ArrayList&lt;&gt;(descriptions);
+     *       Collections.reverse(ordered);
+     *       return ordered;
+     *     }
+     *   }
+     * }
+     *     
+     * public static main() {
+     *   new JUnitCore().run(Request.aClass(AllTests.class).orderWith(reverse()));
+     * }
+     * </pre>
+     *
+     * @return a Request with ordered Tests
+     * @since 4.13
+     */
+    public Request orderWith(Ordering ordering) {
+        return new OrderingRequest(this, ordering);
+    }
 }

src/main/java/org/junit/runner/manipulation/Alphanumeric.java

@@ -0,0 +1,27 @@
+package org.junit.runner.manipulation;
+
+import java.util.Comparator;
+
+import org.junit.runner.Description;
+
+/**
+ * A sorter that orders tests alphanumerically by test name.
+ *
+ * @since 4.13
+ */
+public final class Alphanumeric extends Sorter implements Ordering.Factory {
+
+    public Alphanumeric() {
+        super(COMPARATOR);
+    }
+
+    public Ordering create(Context context) {
+        return this;
+    }
+
+    private static final Comparator<Description> COMPARATOR = new Comparator<Description>() {
+        public int compare(Description o1, Description o2) {
+            return o1.getDisplayName().compareTo(o2.getDisplayName());
+        }
+    };
+}

src/main/java/org/junit/runner/manipulation/InvalidOrderingException.java

@@ -0,0 +1,21 @@
+package org.junit.runner.manipulation;
+
+/**
+ * Thrown when an ordering does something invalid (like remove or add children)
+ *
+ * @since 4.13
+ */
+public class InvalidOrderingException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public InvalidOrderingException() {
+    }
+
+    public InvalidOrderingException(String message) {
+        super(message);
+    }
+
+    public InvalidOrderingException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

src/main/java/org/junit/runner/manipulation/Orderable.java

@@ -0,0 +1,21 @@
+package org.junit.runner.manipulation;
+
+/**
+ * Interface for runners that allow ordering of tests.
+ *
+ * <p>Beware of using this interface to cope with order dependencies between tests.
+ * Tests that are isolated from each other are less expensive to maintain and
+ * can be run individually.
+ *
+ * @since 4.13
+ */
+public interface Orderable extends Sortable {
+
+    /**
+     * Orders the tests using <code>orderer</code>
+     *
+     * @throws InvalidOrderingException if orderer does something invalid (like remove or add
+     * children)
+     */
+    void order(Orderer orderer) throws InvalidOrderingException;
+}