[analyzer,cfe,dart2js] Add marker.options files to automate running id tests on all configurations

Change-Id: Ib5eb7f3967634721047f8dcc57b9e6c350b91a41
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/136226
Reviewed-by: Dmitry Stefantsov <dmitryas@google.com>
Reviewed-by: Paul Berry <paulberry@google.com>
Commit-Queue: Johnni Winther <johnniwinther@google.com>

Author: johnniwinther

pkg/_fe_analyzer_shared/lib/src/testing/id_testing.dart

@@ -628,20 +628,95 @@ typedef Future<Map<String, TestResult<T>>> RunTestFunction<T>(TestData testData,
     Map<String, List<String>> skipMap,
     Uri nullUri});
 
+class MarkerOptions {
+  final Map<String, Uri> markers;
+
+  MarkerOptions.internal(this.markers);
+
+  factory MarkerOptions.fromFile(File file, {bool isUsingShards: false}) {
+    File script = new File.fromUri(Platform.script);
+    if (!file.existsSync()) {
+      throw new ArgumentError("Marker option file '$file' doesn't exist.");
+    }
+
+    Map<String, Uri> markers = {};
+    String text = file.readAsStringSync();
+    bool isScriptFound = false;
+    for (String line in text.split('\n')) {
+      line = line.trim();
+      if (line.isEmpty || line.startsWith('#')) continue;
+      int eqPos = line.indexOf('=');
+      if (eqPos == -1) {
+        throw new ArgumentError(
+            "Unsupported marker option '$line' in ${file.uri}");
+      }
+      String marker = line.substring(0, eqPos);
+      String tester = line.substring(eqPos + 1);
+      File testerFile = new File(tester);
+      if (!testerFile.existsSync()) {
+        throw new ArgumentError(
+            "Tester '$tester' does not exist for marker '$marker' in "
+            "${file.uri}");
+      }
+      if (markers.containsKey(marker)) {
+        throw new ArgumentError("Duplicate marker '$marker' in ${file.uri}");
+      }
+      markers[marker] = testerFile.uri;
+      if (testerFile.absolute.uri == script.absolute.uri) {
+        isScriptFound = true;
+      }
+    }
+    if (!isUsingShards && !isScriptFound) {
+      throw new ArgumentError(
+          "Script '${script.uri}' not found in ${file.uri}");
+    }
+    return new MarkerOptions.internal(markers);
+  }
+
+  Iterable<String> get supportedMarkers => markers.keys;
+
+  Future<void> runAll(List<String> args) async {
+    Set<Uri> testers = markers.values.toSet();
+    bool allOk = true;
+    for (Uri tester in testers) {
+      print('================================================================');
+      print('Running tester: $tester ${args.join(' ')}');
+      print('================================================================');
+      Process process = await Process.start(
+          Platform.resolvedExecutable, [tester.toString(), ...args],
+          mode: ProcessStartMode.inheritStdio);
+      if (await process.exitCode != 0) {
+        allOk = false;
+      }
+    }
+    if (!allOk) {
+      throw "Error(s) occurred.";
+    }
+  }
+}
+
 /// Check code for all tests in [dataDir] using [runTest].
 Future<void> runTests<T>(Directory dataDir,
     {List<String> args: const <String>[],
     int shards: 1,
     int shardIndex: 0,
     void onTest(Uri uri),
-    Iterable<String> supportedMarkers,
     Uri createUriForFileName(String fileName),
     void onFailure(String message),
     RunTestFunction<T> runTest,
     List<String> skipList,
     Map<String, List<String>> skipMap}) async {
+  File markerOptionsFile =
+      new File.fromUri(dataDir.uri.resolve('marker.options'));
+  MarkerOptions markerOptions =
+      new MarkerOptions.fromFile(markerOptionsFile, isUsingShards: shards != 1);
   // TODO(johnniwinther): Support --show to show actual data for an input.
   args = args.toList();
+  bool runAll = args.remove('--run-all');
+  if (runAll) {
+    await markerOptions.runAll(args);
+    return;
+  }
   bool verbose = args.remove('-v');
   bool succinct = args.remove('-s');
   bool shouldContinue = args.remove('-c');
@@ -653,8 +728,11 @@ Future<void> runTests<T>(Directory dataDir,
 
   String relativeDir = dataDir.uri.path.replaceAll(Uri.base.path, '');
   print('Data dir: ${relativeDir}');
-  List<FileSystemEntity> entities =
-      dataDir.listSync().where((entity) => !entity.path.endsWith('~')).toList();
+  List<FileSystemEntity> entities = dataDir
+      .listSync()
+      .where((entity) =>
+          !entity.path.endsWith('~') && !entity.path.endsWith('marker.options'))
+      .toList();
   if (shards > 1) {
     int start = entities.length * shardIndex ~/ shards;
     int end = entities.length * (shardIndex + 1) ~/ shards;
@@ -688,7 +766,7 @@ Future<void> runTests<T>(Directory dataDir,
     }
 
     TestData testData = computeTestData(entity,
-        supportedMarkers: supportedMarkers,
+        supportedMarkers: markerOptions.supportedMarkers,
         createTestUri: createTestUri,
         onFailure: onFailure);
     print('Test: ${testData.testFileUri}');

pkg/_fe_analyzer_shared/test/constants/data/marker.options

@@ -0,0 +1,3 @@
+cfe=pkg/front_end/test/id_tests/constant_test.dart
+analyzer=pkg/analyzer/test/id_tests/constant_test.dart
+dart2js=tests/compiler/dart2js/model/cfe_constant_test.dart

pkg/_fe_analyzer_shared/test/flow_analysis/assigned_variables/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/assigned_variables_test.dart
+analyzer=pkg/analyzer/test/id_tests/assigned_variables_test.dart

pkg/_fe_analyzer_shared/test/flow_analysis/definite_assignment/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/definite_assignment_test.dart
+analyzer=pkg/analyzer/test/id_tests/definite_assignment_test.dart

pkg/_fe_analyzer_shared/test/flow_analysis/nullability/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/nullability_test.dart
+analyzer=pkg/analyzer/test/id_tests/nullability_test.dart

pkg/_fe_analyzer_shared/test/flow_analysis/reachability/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/reachability_test.dart
+analyzer=pkg/analyzer/test/id_tests/reachability_test.dart

pkg/_fe_analyzer_shared/test/flow_analysis/type_promotion/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/type_promotion_test.dart
+analyzer=pkg/analyzer/test/id_tests/type_promotion_test.dart

pkg/_fe_analyzer_shared/test/inheritance/data/marker.options

@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/inheritance_test.dart
+cfe:builder=pkg/front_end/test/id_tests/inheritance_test.dart

pkg/analyzer/test/id_tests/assigned_variables_test.dart

@@ -23,7 +23,6 @@ main(List<String> args) async {
   await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
     return runTests<_Data>(dataDir,
         args: args,
-        supportedMarkers: cfeAnalyzerMarkers,
         createUriForFileName: createUriForFileName,
         onFailure: onFailure,
         runTest: runTestFor(

pkg/analyzer/test/id_tests/constant_test.dart

@@ -22,7 +22,6 @@ main(List<String> args) async {
   await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
     return runTests<String>(dataDir,
         args: args,
-        supportedMarkers: sharedMarkers,
         createUriForFileName: createUriForFileName,
         onFailure: onFailure,
         runTest: runTestFor(