Skip to content

Commit a92ea27

Browse files
winglamwinglam
committed
Enabling Surefire to run test classes and test methods in any order specified by a runOrder
1 parent d5beee5 commit a92ea27

File tree

10 files changed

+288
-9
lines changed

10 files changed

+288
-9
lines changed

‎surefire-api/src/main/java/org/apache/maven/surefire/api/testset/TestListResolver.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
import java.util.ArrayList;
2323
import java.util.Collection;
2424
import java.util.Collections;
25+
import java.util.HashMap;
2526
import java.util.LinkedHashSet;
27+
import java.util.Map;
2628
import java.util.Set;
2729

2830
import static java.util.Collections.unmodifiableSet;
@@ -61,6 +63,8 @@ public class TestListResolver
6163

6264
private final boolean hasExcludedMethodPatterns;
6365

66+
private final Map<String, Integer> patternMapper = new HashMap<String, Integer>();
67+
6468
public TestListResolver( Collection<String> tests )
6569
{
6670
final IncludedExcludedPatterns patterns = new IncludedExcludedPatterns();
@@ -208,6 +212,7 @@ public boolean shouldRun( String testClassFile, String methodName )
208212
else
209213
{
210214
boolean shouldRun = false;
215+
ResolvedTest matchedFilter = null;
211216

212217
if ( getIncludedPatterns().isEmpty() )
213218
{
@@ -220,6 +225,7 @@ public boolean shouldRun( String testClassFile, String methodName )
220225
if ( filter.matchAsInclusive( testClassFile, methodName ) )
221226
{
222227
shouldRun = true;
228+
matchedFilter = filter;
223229
break;
224230
}
225231
}
@@ -236,6 +242,15 @@ public boolean shouldRun( String testClassFile, String methodName )
236242
}
237243
}
238244
}
245+
246+
if ( shouldRun )
247+
{
248+
String test = testClassFile + "#" + methodName;
249+
if ( ! this.patternMapper.containsKey( test ) )
250+
{
251+
this.patternMapper.put( test, new ArrayList<>( this.includedPatterns ).indexOf( matchedFilter ) );
252+
}
253+
}
239254
return shouldRun;
240255
}
241256
}
@@ -514,4 +529,24 @@ private static boolean haveMethodPatterns( Set<ResolvedTest> patterns )
514529
}
515530
return false;
516531
}
532+
533+
public Integer testOrderComparator( String className1, String className2, String methodName1, String methodName2 )
534+
{
535+
String classFileName1 = toClassFileName( className1 );
536+
String classFileName2 = toClassFileName( className2 );
537+
boolean shouldRunMethodName1 = shouldRun( classFileName1 , methodName1 );
538+
boolean shouldRunMethodName2 = shouldRun( classFileName2 , methodName2 );
539+
if ( ! shouldRunMethodName1 )
540+
{
541+
return -1;
542+
}
543+
if ( ! shouldRunMethodName2 )
544+
{
545+
return 1;
546+
}
547+
548+
String test1 = classFileName1 + "#" + methodName1;
549+
String test2 = classFileName2 + "#" + methodName2;
550+
return patternMapper.get( test1 ) - patternMapper.get( test2 );
551+
}
517552
}

‎surefire-api/src/main/java/org/apache/maven/surefire/api/util/DefaultRunOrderCalculator.java‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121

2222
import org.apache.maven.surefire.api.runorder.RunEntryStatisticsMap;
2323
import org.apache.maven.surefire.api.testset.RunOrderParameters;
24+
import org.apache.maven.surefire.api.testset.TestListResolver;
2425

2526
import java.util.ArrayList;
27+
import java.util.Arrays;
2628
import java.util.Calendar;
2729
import java.util.Collections;
2830
import java.util.Comparator;
@@ -48,6 +50,8 @@ public class DefaultRunOrderCalculator
4850

4951
private final Random random;
5052

53+
private final TestListResolver testListResolver;
54+
5155
public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int threadCount )
5256
{
5357
this.runOrderParameters = runOrderParameters;
@@ -61,6 +65,14 @@ public DefaultRunOrderCalculator( RunOrderParameters runOrderParameters, int thr
6165
runOrderParameters.setRunOrderRandomSeed( runOrderRandomSeed );
6266
}
6367
this.random = new Random( runOrderRandomSeed );
68+
if ( RunOrder.TESTORDER.equals( getRunOrderMethod() ) )
69+
{
70+
this.testListResolver = getTestListResolver();
71+
}
72+
else
73+
{
74+
this.testListResolver = null;
75+
}
6476
}
6577

6678
@Override
@@ -78,9 +90,80 @@ public TestsToRun orderTestClasses( TestsToRun scannedClasses )
7890
return new TestsToRun( new LinkedHashSet<>( result ) );
7991
}
8092

93+
@Override
94+
public Comparator<String> comparatorForTestMethods()
95+
{
96+
RunOrder methodRunOrder = getRunOrderMethod();
97+
if ( RunOrder.TESTORDER.equals( methodRunOrder ) )
98+
{
99+
return new Comparator<String>()
100+
{
101+
@Override
102+
public int compare( String o1, String o2 )
103+
{
104+
String[] classAndMethod1 = getClassAndMethod( o1 );
105+
String className1 = classAndMethod1[0];
106+
String methodName1 = classAndMethod1[1];
107+
String[] classAndMethod2 = getClassAndMethod( o2 );
108+
String className2 = classAndMethod2[0];
109+
String methodName2 = classAndMethod2[1];
110+
return testListResolver.testOrderComparator( className1, className2, methodName1, methodName2 );
111+
}
112+
};
113+
}
114+
else
115+
{
116+
return null;
117+
}
118+
}
119+
120+
public TestListResolver getTestListResolver()
121+
{
122+
String orderParam = System.getProperty( "test" );
123+
if ( orderParam == null )
124+
{
125+
throw new IllegalStateException( "TestListResolver in RunOrderCalculator should be used only when "
126+
+ "system property -Dtest is set and runOrder is testorder" );
127+
}
128+
return new TestListResolver( Arrays.asList( orderParam.split( "," ) ) );
129+
}
130+
131+
public String[] getClassAndMethod( String request )
132+
{
133+
String[] classAndMethod = { request, request };
134+
if ( request.contains( "(" ) )
135+
{
136+
String[] nameSplit1 = request.split( "\\(" );
137+
classAndMethod[0] = nameSplit1[1].substring( 0, nameSplit1[1].length() - 1 );
138+
classAndMethod[1] = nameSplit1[0];
139+
}
140+
return classAndMethod;
141+
}
142+
143+
private RunOrder getRunOrderMethod()
144+
{
145+
if ( runOrder.length > 1 && Arrays.asList( runOrder ).contains( RunOrder.TESTORDER ) )
146+
{
147+
// Use of testorder and other runOrders are currently not supported
148+
throw new IllegalStateException( "Expected only testorder. Got: " + runOrder.length );
149+
}
150+
return runOrder[0];
151+
}
152+
81153
private void orderTestClasses( List<Class<?>> testClasses, RunOrder runOrder )
82154
{
83-
if ( RunOrder.RANDOM.equals( runOrder ) )
155+
if ( RunOrder.TESTORDER.equals( runOrder ) )
156+
{
157+
Collections.sort( testClasses, new Comparator<Class<?>>()
158+
{
159+
@Override
160+
public int compare( Class<?> o1, Class<?> o2 )
161+
{
162+
return testListResolver.testOrderComparator( o1.getName(), o2.getName(), null, null );
163+
}
164+
} );
165+
}
166+
else if ( RunOrder.RANDOM.equals( runOrder ) )
84167
{
85168
Collections.shuffle( testClasses, random );
86169
}

‎surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrder.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public class RunOrder
4444

4545
public static final RunOrder FAILEDFIRST = new RunOrder( "failedfirst" );
4646

47+
public static final RunOrder TESTORDER = new RunOrder( "testorder" );
48+
4749
public static final RunOrder[] DEFAULT = new RunOrder[]{ FILESYSTEM };
4850

4951
/**
@@ -108,7 +110,8 @@ private static String createMessageForMissingRunOrder( String name )
108110

109111
private static RunOrder[] values()
110112
{
111-
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST };
113+
return new RunOrder[]{ ALPHABETICAL, FILESYSTEM, HOURLY, RANDOM, REVERSE_ALPHABETICAL, BALANCED, FAILEDFIRST,
114+
TESTORDER };
112115
}
113116

114117
public static String asString( RunOrder[] runOrder )

‎surefire-api/src/main/java/org/apache/maven/surefire/api/util/RunOrderCalculator.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,14 @@
1919
* under the License.
2020
*/
2121

22+
import java.util.Comparator;
23+
2224
/**
2325
* @author Kristian Rosenvold
2426
*/
2527
public interface RunOrderCalculator
2628
{
2729
TestsToRun orderTestClasses( TestsToRun scannedClasses );
30+
31+
Comparator<String> comparatorForTestMethods();
2832
}

‎surefire-api/src/test/java/org/apache/maven/surefire/api/testset/TestListResolverTest.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.HashSet;
2828
import java.util.Iterator;
2929
import java.util.LinkedHashSet;
30+
import java.util.List;
3031
import java.util.Set;
3132

3233
import static java.util.Collections.addAll;
@@ -502,4 +503,68 @@ private static Set<ResolvedTest> resolveClass( String patterns )
502503
}
503504
return resolved;
504505
}
506+
507+
public void testOrderComparatorTest()
508+
{
509+
List<String> orderParamList = new ArrayList<String>();
510+
orderParamList.add( "TestClass1#testa2d" );
511+
orderParamList.add( "TestClass1#testabc" );
512+
orderParamList.add( "TestClass1#testa1b" );
513+
orderParamList.add( "TestClass2#testa1b" );
514+
orderParamList.add( "TestClass2#testaBc" );
515+
TestListResolver tlr = new TestListResolver( orderParamList );
516+
String className = "TestClass1";
517+
String className2 = "TestClass2";
518+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa1b" ) < 0 );
519+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testabc" ) < 0 );
520+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testabc" ) > 0 );
521+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testaBc" ) > 0 );
522+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa3d", "testa1b" ) < 0 );
523+
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testa1b" ) < 0 );
524+
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testaBc", "testa1b" ) > 0 );
525+
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa3d", "testa1b" ) < 0 );
526+
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa2d", "testabc" ) > 0 );
527+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa2d", "testa2d" ) == 0 );
528+
}
529+
530+
public void testRegexMethodOrderComparator()
531+
{
532+
List<String> orderParamList = new ArrayList<String>();
533+
orderParamList.add( "TestClass1#testa?c" );
534+
orderParamList.add( "TestClass1#testa?b" );
535+
orderParamList.add( "TestClass2#test?1*" );
536+
orderParamList.add( "!TestClass1#testa4b" );
537+
orderParamList.add( "!TestClass2#test11MyTest" );
538+
TestListResolver tlr = new TestListResolver( orderParamList );
539+
String className = "TestClass1";
540+
String className2 = "TestClass2";
541+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testabc", "testa1b" ) < 0 );
542+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testaBc", "testa2b" ) < 0 );
543+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa3c" ) > 0 );
544+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1b", "testa4b" ) > 0 );
545+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa4b", "testabc" ) < 0 );
546+
assertTrue( ( int ) tlr.testOrderComparator( className, className2, "testa1b", "test1123" ) < 0 );
547+
assertTrue( ( int ) tlr.testOrderComparator( className2, className, "testa1b", "testa1b" ) > 0 );
548+
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "testa1b", "test1123" ) == 0 );
549+
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test1123", "test11MyTest" ) > 0 );
550+
assertTrue( ( int ) tlr.testOrderComparator( className2, className2, "test11MyTest", "test456" ) < 0 );
551+
assertTrue( ( int ) tlr.testOrderComparator( className, className, "testa1c", "testa1c" ) == 0 );
552+
}
553+
554+
public void testRegexClassOrderComparator()
555+
{
556+
List<String> orderParamList = new ArrayList<String>();
557+
orderParamList.add( "My2*Test.java" );
558+
orderParamList.add( "???My1*Test" );
559+
orderParamList.add( "!abcMy1PeaceTest" );
560+
TestListResolver tlr = new TestListResolver( orderParamList );
561+
String className = "My2ConnectTest";
562+
String className2 = "456My1ConnectTest";
563+
String className3 = "abcMy1PeaceTest";
564+
assertTrue( ( int ) tlr.testOrderComparator( className, className2, null, null ) < 0 );
565+
assertTrue( ( int ) tlr.testOrderComparator( className2, className, null, null ) > 0 );
566+
assertTrue( ( int ) tlr.testOrderComparator( className3, className2, null, null ) < 0 );
567+
assertTrue( ( int ) tlr.testOrderComparator( className, className3, null, null ) > 0 );
568+
assertTrue( ( int ) tlr.testOrderComparator( className, className, null, null ) == 0 );
569+
}
505570
}

‎surefire-api/src/test/java/org/apache/maven/surefire/api/util/RunOrderCalculatorTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@
1919
* under the License.
2020
*/
2121

22+
import java.util.Arrays;
23+
import java.util.Comparator;
2224
import java.util.LinkedHashSet;
25+
import java.util.List;
2326
import java.util.Set;
2427

2528
import org.apache.maven.surefire.api.testset.RunOrderParameters;
@@ -59,4 +62,52 @@ static class B
5962
{
6063

6164
}
65+
66+
public void testOrderTestMethods()
67+
{
68+
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
69+
System.setProperty( "test", "TestClass#a2d,TestClass#aBc,TestClass#abc,TestClass#a1b" );
70+
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
71+
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
72+
String[] strArray = { "abc(TestClass)", "a1b(TestClass)", "a2d(TestClass)", "aBc(TestClass)" };
73+
List<String> actual = Arrays.asList( strArray );
74+
actual.sort( testOrderRunOrderComparator );
75+
String[] strArray2 = { "a2d(TestClass)", "aBc(TestClass)", "abc(TestClass)", "a1b(TestClass)" };
76+
List<String> expected = Arrays.asList( strArray2 );
77+
assertEquals( actual, expected );
78+
}
79+
80+
public void testOrderTestClassesAndMethods()
81+
{
82+
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
83+
System.setProperty( "test", "TestClass1#a2d,TestClass2#aBc,TestClass2#abc,TestClass2#a1b" );
84+
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
85+
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
86+
String[] strArray = { "abc(TestClass2)", "a1b(TestClass2)", "a2d(TestClass1)", "aBc(TestClass2)" };
87+
List<String> actual = Arrays.asList( strArray );
88+
actual.sort( testOrderRunOrderComparator );
89+
String[] strArray2 = { "a2d(TestClass1)", "aBc(TestClass2)", "abc(TestClass2)", "a1b(TestClass2)" };
90+
List<String> expected = Arrays.asList( strArray2 );
91+
assertEquals( actual, expected );
92+
}
93+
94+
public void testOrderTestRegexClassesAndMethods()
95+
{
96+
RunOrderParameters runOrderParameters = new RunOrderParameters( "testorder" , null );
97+
System.setProperty( "test", "Amber*Test#a?c,My???Test#test*" );
98+
DefaultRunOrderCalculator runOrderCalculator = new DefaultRunOrderCalculator( runOrderParameters, 1 );
99+
Comparator<String> testOrderRunOrderComparator = runOrderCalculator.comparatorForTestMethods();
100+
String[] strArray = { "abc(AmberGoodTest)",
101+
"testabc(MyabcTest)",
102+
"a2c(AmberBadTest)",
103+
"testefg(MyefgTest)",
104+
"aBc(AmberGoodTest)" };
105+
List<String> actual = Arrays.asList( strArray );
106+
actual.sort( testOrderRunOrderComparator );
107+
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 0 ) )[0].substring( 0, 5 ), "Amber" );
108+
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 1 ) )[0].substring( 0, 5 ), "Amber" );
109+
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 2 ) )[0].substring( 0, 5 ), "Amber" );
110+
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 3 ) )[0].substring( 0, 2 ), "My" );
111+
assertEquals( runOrderCalculator.getClassAndMethod( actual.get( 4 ) )[0].substring( 0, 2 ), "My" );
112+
}
62113
}

0 commit comments

Comments
 (0)