[infra] Support multiple named configurations in test infrastructure

This change allows to run test.dart and pkg/test_runner with multiple arguments
for the -n option to run tests for multiple configurations with one invocation.

Change-Id: If62e0bfc364460fa415c7f700f7e449b0de56987
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/122395
Commit-Queue: Karl Klose <karlklose@google.com>
Reviewed-by: William Hesse <whesse@google.com>

Author: karlklose

pkg/test_runner/lib/src/options.dart

@@ -679,10 +679,87 @@ compiler.''',
     var progress = Progress.find(data["progress"] as String);
     var nnbdMode = NnbdMode.find(data["nnbd"] as String);
 
+    void addConfiguration(Configuration innerConfiguration,
+        [String namedConfiguration]) {
+      var configuration = TestConfiguration(
+          configuration: innerConfiguration,
+          progress: progress,
+          selectors: selectors,
+          testList: data["test_list_contents"] as List<String>,
+          repeat: data["repeat"] as int,
+          batch: !(data["noBatch"] as bool),
+          batchDart2JS: data["dart2js_batch"] as bool,
+          copyCoreDumps: data["copy_coredumps"] as bool,
+          isVerbose: data["verbose"] as bool,
+          listTests: data["list"] as bool,
+          listStatusFiles: data["list_status_files"] as bool,
+          cleanExit: data["clean_exit"] as bool,
+          silentFailures: data["silent_failures"] as bool,
+          printTiming: data["time"] as bool,
+          printReport: data["report"] as bool,
+          reportInJson: data["report_in_json"] as bool,
+          resetBrowser: data["reset_browser_configuration"] as bool,
+          skipCompilation: data["skip_compilation"] as bool,
+          writeDebugLog: data["write_debug_log"] as bool,
+          writeResults: data["write_results"] as bool,
+          writeLogs: data["write_logs"] as bool,
+          drtPath: data["drt"] as String,
+          chromePath: data["chrome"] as String,
+          safariPath: data["safari"] as String,
+          firefoxPath: data["firefox"] as String,
+          dartPath: data["dart"] as String,
+          dartPrecompiledPath: data["dart_precompiled"] as String,
+          genSnapshotPath: data["gen-snapshot"] as String,
+          keepGeneratedFiles: data["keep_generated_files"] as bool,
+          taskCount: data["tasks"] as int,
+          shardCount: data["shards"] as int,
+          shard: data["shard"] as int,
+          stepName: data["step_name"] as String,
+          testServerPort: data["test_server_port"] as int,
+          testServerCrossOriginPort:
+              data['test_server_cross_origin_port'] as int,
+          testDriverErrorPort: data["test_driver_error_port"] as int,
+          localIP: data["local_ip"] as String,
+          sharedOptions: sharedOptions,
+          packages: data["packages"] as String,
+          packageRoot: data["package_root"] as String,
+          suiteDirectory: data["suite_dir"] as String,
+          outputDirectory: data["output_directory"] as String,
+          reproducingArguments:
+              _reproducingCommand(data, namedConfiguration != null),
+          fastTestsOnly: data["fast_tests"] as bool,
+          printPassingStdout: data["print_passing_stdout"] as bool);
+
+      if (configuration.validate()) {
+        result.add(configuration);
+      }
+    }
+
+    String namedConfigurationOption = data["named_configuration"] as String;
+    if (namedConfigurationOption != null) {
+      List<String> namedConfigurations = namedConfigurationOption.split(',');
+      var testMatrixFile = "tools/bots/test_matrix.json";
+      var testMatrix = TestMatrix.fromPath(testMatrixFile);
+      for (String namedConfiguration in namedConfigurations) {
+        var configuration = testMatrix.configurations.singleWhere(
+            (c) => c.name == namedConfiguration,
+            orElse: () => null);
+        if (configuration == null) {
+          var names = testMatrix.configurations
+              .map((configuration) => configuration.name)
+              .toList();
+          names.sort();
+          _fail('The named configuration "$namedConfiguration" does not exist.'
+              ' The following configurations are available:\n'
+              '  * ${names.join('\n  * ')}');
+        }
+        addConfiguration(configuration);
+      }
+      return result;
+    }
+
     // Expand runtimes.
     for (var runtime in runtimes) {
-      // Start installing the runtime if needed.
-
       // Expand architectures.
       var architectures = data["arch"] as String;
       if (architectures == "all") {
@@ -700,82 +777,28 @@ compiler.''',
           for (var modeName in modes.split(",")) {
             var mode = Mode.find(modeName);
             var system = System.find(data["system"] as String);
-            var namedConfiguration =
-                _namedConfiguration(data["named_configuration"] as String);
-            var innerConfiguration = namedConfiguration ??
-                Configuration("custom configuration", architecture, compiler,
-                    mode, runtime, system,
-                    nnbdMode: nnbdMode,
-                    timeout: data["timeout"] as int,
-                    enableAsserts: data["enable_asserts"] as bool,
-                    useAnalyzerCfe: data["use_cfe"] as bool,
-                    useAnalyzerFastaParser:
-                        data["analyzer_use_fasta_parser"] as bool,
-                    useBlobs: data["use_blobs"] as bool,
-                    useElf: data["use_elf"] as bool,
-                    useSdk: data["use_sdk"] as bool,
-                    useHotReload: data["hot_reload"] as bool,
-                    useHotReloadRollback: data["hot_reload_rollback"] as bool,
-                    isHostChecked: data["host_checked"] as bool,
-                    isCsp: data["csp"] as bool,
-                    isMinified: data["minified"] as bool,
-                    vmOptions: vmOptions,
-                    dart2jsOptions: dart2jsOptions,
-                    experiments: experiments,
-                    babel: data['babel'] as String,
-                    builderTag: data["builder_tag"] as String);
-            var configuration = TestConfiguration(
-                configuration: innerConfiguration,
-                progress: progress,
-                selectors: selectors,
-                testList: data["test_list_contents"] as List<String>,
-                repeat: data["repeat"] as int,
-                batch: !(data["noBatch"] as bool),
-                batchDart2JS: data["dart2js_batch"] as bool,
-                copyCoreDumps: data["copy_coredumps"] as bool,
-                isVerbose: data["verbose"] as bool,
-                listTests: data["list"] as bool,
-                listStatusFiles: data["list_status_files"] as bool,
-                cleanExit: data["clean_exit"] as bool,
-                silentFailures: data["silent_failures"] as bool,
-                printTiming: data["time"] as bool,
-                printReport: data["report"] as bool,
-                reportInJson: data["report_in_json"] as bool,
-                resetBrowser: data["reset_browser_configuration"] as bool,
-                skipCompilation: data["skip_compilation"] as bool,
-                writeDebugLog: data["write_debug_log"] as bool,
-                writeResults: data["write_results"] as bool,
-                writeLogs: data["write_logs"] as bool,
-                drtPath: data["drt"] as String,
-                chromePath: data["chrome"] as String,
-                safariPath: data["safari"] as String,
-                firefoxPath: data["firefox"] as String,
-                dartPath: data["dart"] as String,
-                dartPrecompiledPath: data["dart_precompiled"] as String,
-                genSnapshotPath: data["gen-snapshot"] as String,
-                keepGeneratedFiles: data["keep_generated_files"] as bool,
-                taskCount: data["tasks"] as int,
-                shardCount: data["shards"] as int,
-                shard: data["shard"] as int,
-                stepName: data["step_name"] as String,
-                testServerPort: data["test_server_port"] as int,
-                testServerCrossOriginPort:
-                    data['test_server_cross_origin_port'] as int,
-                testDriverErrorPort: data["test_driver_error_port"] as int,
-                localIP: data["local_ip"] as String,
-                sharedOptions: sharedOptions,
-                packages: data["packages"] as String,
-                packageRoot: data["package_root"] as String,
-                suiteDirectory: data["suite_dir"] as String,
-                outputDirectory: data["output_directory"] as String,
-                reproducingArguments:
-                    _reproducingCommand(data, namedConfiguration != null),
-                fastTestsOnly: data["fast_tests"] as bool,
-                printPassingStdout: data["print_passing_stdout"] as bool);
-
-            if (configuration.validate()) {
-              result.add(configuration);
-            }
+            var configuration = Configuration("custom configuration",
+                architecture, compiler, mode, runtime, system,
+                nnbdMode: nnbdMode,
+                timeout: data["timeout"] as int,
+                enableAsserts: data["enable_asserts"] as bool,
+                useAnalyzerCfe: data["use_cfe"] as bool,
+                useAnalyzerFastaParser:
+                    data["analyzer_use_fasta_parser"] as bool,
+                useBlobs: data["use_blobs"] as bool,
+                useElf: data["use_elf"] as bool,
+                useSdk: data["use_sdk"] as bool,
+                useHotReload: data["hot_reload"] as bool,
+                useHotReloadRollback: data["hot_reload_rollback"] as bool,
+                isHostChecked: data["host_checked"] as bool,
+                isCsp: data["csp"] as bool,
+                isMinified: data["minified"] as bool,
+                vmOptions: vmOptions,
+                dart2jsOptions: dart2jsOptions,
+                experiments: experiments,
+                babel: data['babel'] as String,
+                builderTag: data["builder_tag"] as String);
+            addConfiguration(configuration);
           }
         }
       }
@@ -934,25 +957,6 @@ class OptionParseException implements Exception {
   OptionParseException(this.message);
 }
 
-Configuration _namedConfiguration(String template) {
-  if (template == null) return null;
-
-  var testMatrixFile = "tools/bots/test_matrix.json";
-  var testMatrix = TestMatrix.fromPath(testMatrixFile);
-  var configuration = testMatrix.configurations
-      .singleWhere((c) => c.name == template, orElse: () => null);
-  if (configuration == null) {
-    var names = testMatrix.configurations
-        .map((configuration) => configuration.name)
-        .toList();
-    names.sort();
-    _fail('The named configuration "$template" does not exist. The following '
-        'configurations are available:\n  * ${names.join('\n  * ')}');
-  }
-
-  return configuration;
-}
-
 /// Throws an [OptionParseException] with [message].
 void _fail(String message) {
   throw OptionParseException(message);

pkg/test_runner/lib/src/test_configurations.dart

@@ -45,7 +45,6 @@ final TEST_SUITE_DIRECTORIES = [
 
 Future testConfigurations(List<TestConfiguration> configurations) async {
   var startTime = DateTime.now();
-  var startStopwatch = Stopwatch()..start();
 
   // Extract global options from first configuration.
   var firstConf = configurations[0];
@@ -209,7 +208,7 @@ Future testConfigurations(List<TestConfiguration> configurations) async {
   }
 
   if (firstConf.writeResults) {
-    eventListener.add(ResultWriter(firstConf, startTime, startStopwatch));
+    eventListener.add(ResultWriter(firstConf.outputDirectory));
   }
 
   if (firstConf.copyCoreDumps) {

pkg/test_runner/lib/src/test_progress.dart

@@ -708,15 +708,11 @@ String _buildSummaryEnd(Formatter formatter, int failedTests) {
 /// Writes a results.json file with a line for each test.
 /// Each line is a json map with the test name and result and expected result.
 class ResultWriter extends EventListener {
-  final TestConfiguration _configuration;
   final List<Map> _results = [];
   final List<Map> _logs = [];
   final String _outputDirectory;
-  final Stopwatch _startStopwatch;
-  final DateTime _startTime;
 
-  ResultWriter(this._configuration, this._startTime, this._startStopwatch)
-      : _outputDirectory = _configuration.outputDirectory;
+  ResultWriter(this._outputDirectory);
 
   void allTestsKnown() {
     // Write an empty result log file, that will be overwritten if any tests
@@ -729,10 +725,6 @@ class ResultWriter extends EventListener {
       lines.map((l) => l + '\n').join();
 
   void done(TestCase test) {
-    if (_configuration != test.configuration) {
-      throw Exception("Two configurations in the same run. "
-          "Cannot output results for multiple configurations.");
-    }
     final name = test.displayName;
     final index = name.indexOf('/');
     final suite = name.substring(0, index);
@@ -742,7 +734,7 @@ class ResultWriter extends EventListener {
 
     final record = {
       "name": name,
-      "configuration": _configuration.configuration.name,
+      "configuration": test.configuration.configuration.name,
       "suite": suite,
       "test_name": testName,
       "time_ms": time.inMilliseconds,