[cfe] Support sharding in pkg/testing tests

Splits strong_test into 4 shards. This test took 4+ minutes locally with the fast_strong_test and analyze_test ticking in at ~1:45. This change makes local testing of frontend unittests take 2 instead of 4 minutes in total. Change-Id: Id6015de86d547b209a699b1e5196b3edad1e6977 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/114501 Reviewed-by: Jens Johansen <jensj@google.com>
dart-lang · Aug 27, 2019 · 7dbf8ee · 7dbf8ee
1 parent 6c67280
commit 7dbf8ee
Show file tree

Hide file tree

Showing 28 changed files with 90 additions and 26 deletions.
diff --git a/pkg/dev_compiler/test/sourcemap/sourcemaps_ddc_suite.dart b/pkg/dev_compiler/test/sourcemap/sourcemaps_ddc_suite.dart
@@ -89,4 +89,4 @@ class DevCompilerRunner implements CompilerRunner {
 }
 
 void main(List<String> arguments) =>
-    runMe(arguments, createContext, "testing.json");
+    runMe(arguments, createContext, configurationPath: "testing.json");
diff --git a/pkg/dev_compiler/test/sourcemap/sourcemaps_ddk_suite.dart b/pkg/dev_compiler/test/sourcemap/sourcemaps_ddk_suite.dart
@@ -105,4 +105,4 @@ class DevCompilerRunner implements CompilerRunner {
 }
 
 void main(List<String> arguments) =>
-    runMe(arguments, createContext, "testing.json");
+    runMe(arguments, createContext, configurationPath: "testing.json");
diff --git a/pkg/dev_compiler/test/sourcemap/stacktrace_ddc_suite.dart b/pkg/dev_compiler/test/sourcemap/stacktrace_ddc_suite.dart
@@ -22,4 +22,4 @@ class StackTraceContext extends ChainContextWithCleanupHelper {
 }
 
 void main(List<String> arguments) =>
-    runMe(arguments, createContext, "testing.json");
+    runMe(arguments, createContext, configurationPath: "testing.json");
diff --git a/pkg/dev_compiler/test/sourcemap/stacktrace_ddk_suite.dart b/pkg/dev_compiler/test/sourcemap/stacktrace_ddk_suite.dart
@@ -29,4 +29,4 @@ class StackTraceContext extends ChainContextWithCleanupHelper
 }
 
 void main(List<String> arguments) =>
-    runMe(arguments, createContext, "testing.json");
+    runMe(arguments, createContext, configurationPath: "testing.json");
diff --git a/pkg/front_end/test/fasta/expression_test.dart b/pkg/front_end/test/fasta/expression_test.dart
@@ -433,4 +433,4 @@ Future<Context> createContext(
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/fasta/fast_strong_test.dart b/pkg/front_end/test/fasta/fast_strong_test.dart
@@ -18,4 +18,5 @@ Future<FastaContext> createContext(
 }
 
 main([List<String> arguments = const []]) => runMe(arguments, createContext,
-    "../../testing.json", Platform.script.resolve("strong_test.dart"));
+    configurationPath: "../../testing.json",
+    me: Platform.script.resolve("strong_tester.dart"));
diff --git a/pkg/front_end/test/fasta/incremental_test.dart b/pkg/front_end/test/fasta/incremental_test.dart
@@ -246,4 +246,4 @@ Future<Context> createContext(
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/fasta/messages_test.dart b/pkg/front_end/test/fasta/messages_test.dart
@@ -663,4 +663,4 @@ String relativize(Uri uri) {
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/fasta/outline_test.dart b/pkg/front_end/test/fasta/outline_test.dart
@@ -14,4 +14,4 @@ Future<FastaContext> createContext(
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/fasta/parser/parser_suite.dart b/pkg/front_end/test/fasta/parser/parser_suite.dart
@@ -41,4 +41,4 @@ class Parse extends Step<ScannedFile, Null, ChainContext> {
 }
 
 main(List<String> arguments) =>
-    runMe(arguments, createContext, "../../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../../testing.json");
diff --git a/pkg/front_end/test/fasta/scanner/scanner_suite.dart b/pkg/front_end/test/fasta/scanner/scanner_suite.dart
@@ -20,4 +20,4 @@ class ScannerContext extends ChainContext {
 }
 
 main(List<String> arguments) =>
-    runMe(arguments, createContext, "../../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../../testing.json");
diff --git a/pkg/front_end/test/fasta/strong1_test.dart b/pkg/front_end/test/fasta/strong1_test.dart
@@ -0,0 +1,9 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE.md file.
+
+import 'strong_tester.dart';
+
+main(List<String> arguments) {
+  internalMain(arguments: arguments, shards: shardCount, shard: 0);
+}
diff --git a/pkg/front_end/test/fasta/strong2_test.dart b/pkg/front_end/test/fasta/strong2_test.dart
@@ -0,0 +1,9 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE.md file.
+
+import 'strong_tester.dart';
+
+main(List<String> arguments) {
+  internalMain(arguments: arguments, shards: shardCount, shard: 1);
+}
diff --git a/pkg/front_end/test/fasta/strong3_test.dart b/pkg/front_end/test/fasta/strong3_test.dart
@@ -0,0 +1,9 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE.md file.
+
+import 'strong_tester.dart';
+
+main(List<String> arguments) {
+  internalMain(arguments: arguments, shards: shardCount, shard: 2);
+}
diff --git a/pkg/front_end/test/fasta/strong4_test.dart b/pkg/front_end/test/fasta/strong4_test.dart
@@ -0,0 +1,9 @@
+// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE.md file.
+
+import 'strong_tester.dart';
+
+main(List<String> arguments) {
+  internalMain(arguments: arguments, shards: shardCount, shard: 3);
+}
diff --git a/pkg/front_end/test/fasta/strong_test.dart → pkg/front_end/test/fasta/strong_tester.dart b/pkg/front_end/test/fasta/strong_test.dart → pkg/front_end/test/fasta/strong_tester.dart
@@ -5,14 +5,27 @@
 library fasta.test.strong_test;
 
 import 'dart:async' show Future;
+import 'dart:io' show Platform;
 
 import 'testing/suite.dart';
 
+const int shardCount = 4;
+
 Future<FastaContext> createContext(
     Chain suite, Map<String, String> environment) {
   environment[ENABLE_FULL_COMPILE] = "";
   return FastaContext.create(suite, environment);
 }
 
-main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+main(List<String> arguments) {
+  internalMain(arguments: arguments);
+}
+
+internalMain(
+    {List<String> arguments = const [], int shards = 1, int shard = 0}) {
+  runMe(arguments, createContext,
+      configurationPath: "../../testing.json",
+      me: Platform.script.resolve('strong_tester.dart'),
+      shards: shards,
+      shard: shard);
+}
diff --git a/pkg/front_end/test/fasta/text_serialization_test.dart b/pkg/front_end/test/fasta/text_serialization_test.dart
@@ -16,4 +16,4 @@ Future<FastaContext> createContext(
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/fasta/type_promotion_look_ahead_test.dart b/pkg/front_end/test/fasta/type_promotion_look_ahead_test.dart
@@ -227,4 +227,4 @@ class CheckTypePromotionResult
 }
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../../testing.json");
+    runMe(arguments, createContext, configurationPath: "../../testing.json");
diff --git a/pkg/front_end/test/incremental_bulk_compiler_full.dart b/pkg/front_end/test/incremental_bulk_compiler_full.dart
@@ -29,7 +29,7 @@ import 'package:testing/testing.dart'
 import 'incremental_utils.dart' as util;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/test/incremental_bulk_compiler_smoke_test.dart b/pkg/front_end/test/incremental_bulk_compiler_smoke_test.dart
@@ -9,7 +9,7 @@ import 'package:testing/testing.dart' show Chain, runMe;
 import 'incremental_bulk_compiler_full.dart' show Context;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/test/incremental_load_from_dill_test.dart b/pkg/front_end/test/incremental_load_from_dill_test.dart
@@ -60,7 +60,7 @@ import 'package:front_end/src/fasta/fasta_codes.dart'
     show DiagnosticMessageFromJson, FormattedMessage;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/test/lint_test.dart b/pkg/front_end/test/lint_test.dart
@@ -32,7 +32,7 @@ import 'package:testing/testing.dart'
     show Chain, ChainContext, Result, Step, TestDescription, runMe;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/test/old_dill_test.dart b/pkg/front_end/test/old_dill_test.dart
@@ -20,7 +20,7 @@ Future<Null> main([List<String> arguments = const []]) async {
     await checkDill();
     return null;
   }
-  await runMe(arguments, createContext, "../testing.json");
+  await runMe(arguments, createContext, configurationPath: "../testing.json");
   await checkDill();
 }
 

diff --git a/pkg/front_end/test/spelling_test_not_src_test.dart b/pkg/front_end/test/spelling_test_not_src_test.dart
@@ -11,7 +11,7 @@ import 'spelling_test_base.dart';
 import 'spell_checking_utils.dart' as spell;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/test/spelling_test_src_test.dart b/pkg/front_end/test/spelling_test_src_test.dart
@@ -11,7 +11,7 @@ import 'spelling_test_base.dart';
 import 'spell_checking_utils.dart' as spell;
 
 main([List<String> arguments = const []]) =>
-    runMe(arguments, createContext, "../testing.json");
+    runMe(arguments, createContext, configurationPath: "../testing.json");
 
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {

diff --git a/pkg/front_end/testing.json b/pkg/front_end/testing.json
@@ -71,7 +71,7 @@
     {
       "name": "strong",
       "kind": "Chain",
-      "source": "test/fasta/strong_test.dart",
+      "source": "test/fasta/strong_tester.dart",
       "path": "testcases/",
       "status": "testcases/strong.status",
       "pattern": [

diff --git a/pkg/testing/lib/src/chain.dart b/pkg/testing/lib/src/chain.dart
@@ -111,7 +111,11 @@ abstract class ChainContext {
 
   ExpectationSet get expectationSet => ExpectationSet.Default;
 
-  Future<Null> run(Chain suite, Set<String> selectors) async {
+  Future<Null> run(Chain suite, Set<String> selectors,
+      {int shards = 1, int shard = 0}) async {
+    assert(shards >= 1, "Invalid shards count: $shards");
+    assert(0 <= shard && shard < shards,
+        "Invalid shard index: $shard, not in range [0,$shards[.");
     List<String> partialSelectors = selectors
         .where((s) => s.endsWith('...'))
         .map((s) => s.substring(0, s.length - 3))
@@ -124,6 +128,15 @@ abstract class ChainContext {
     }
     List<TestDescription> descriptions = await stream.toList();
     descriptions.sort();
+    if (shards > 1) {
+      List<TestDescription> shardDescriptions = [];
+      for (int index = 0; index < descriptions.length; index++) {
+        if (index % shards == shard) {
+          shardDescriptions.add(descriptions[index]);
+        }
+      }
+      descriptions = shardDescriptions;
+    }
     Map<TestDescription, Result> unexpectedResults =
         <TestDescription, Result>{};
     Map<TestDescription, Set<Expectation>> unexpectedOutcomes =

diff --git a/pkg/testing/lib/src/run.dart b/pkg/testing/lib/src/run.dart
@@ -55,7 +55,7 @@ Future<TestRoot> computeTestRoot(String configurationPath, Uri base) {
 /// `testing.json` isn't located in the current working directory and is a path
 /// relative to [me] which defaults to `Platform.script`.
 Future<Null> runMe(List<String> arguments, CreateContext f,
-    [String configurationPath, Uri me]) {
+    {String configurationPath, Uri me, int shards = 1, int shard = 0}) {
   me ??= Platform.script;
   return withErrorHandling(() async {
     TestRoot testRoot = await computeTestRoot(configurationPath, me);
@@ -65,7 +65,8 @@ Future<Null> runMe(List<String> arguments, CreateContext f,
       if (me == suite.source) {
         print("Running suite ${suite.name}...");
         ChainContext context = await f(suite, cl.environment);
-        await context.run(suite, new Set<String>.from(cl.selectors));
+        await context.run(suite, new Set<String>.from(cl.selectors),
+            shards: shards, shard: shard);
       }
     }
   });