migrate custom check verification to a test (#4406)

test/all.dart

@@ -21,6 +21,7 @@ import 'validate_no_rule_description_references.dart'
     as validate_no_rule_description_references;
 import 'validate_rule_description_format_test.dart'
     as validate_rule_description_format;
+import 'verify_checks_test.dart' as verify_checks;
 import 'verify_reflective_test_suites.dart' as verify_reflective_test_suites;
 import 'version_test.dart' as version_test;
 
@@ -42,6 +43,7 @@ void main() {
   validate_incompatible_rules.main();
   validate_no_rule_description_references.main();
   validate_rule_description_format.main();
+  verify_checks.main();
   verify_reflective_test_suites.main();
   version_test.main();
 }

test/verify_checks_test.dart

@@ -0,0 +1,17 @@
+// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+import '../tool/checks/driver.dart';
+
+main() {
+  group('custom checks', () {
+    var checkNames = customChecks.map((c) => c.name).join(', ');
+    test(checkNames, () async {
+      var failedChecks = await runChecks();
+      expect(failedChecks, isEmpty);
+    });
+  }, timeout: Timeout(Duration(minutes: 2)));
+}

tool/bot.sh

@@ -80,9 +80,6 @@ else
   # Verify that the libraries are error free.
   dart analyze --fatal-infos .
 
-  # Enforce some linter-specific checks.
-  dart tool/checks/driver.dart
-
   echo ""
 
   # Run tests.

tool/checks/driver.dart

@@ -6,6 +6,7 @@ import 'dart:io' as io;
 
 import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
 import 'package:analyzer/dart/analysis/results.dart';
+import 'package:analyzer/error/error.dart';
 import 'package:analyzer/file_system/file_system.dart';
 import 'package:analyzer/file_system/physical_file_system.dart';
 import 'package:analyzer/src/generated/engine.dart' // ignore: implementation_imports
@@ -18,52 +19,56 @@ import 'package:linter/src/analyzer.dart';
 import 'package:linter/src/formatter.dart';
 import 'package:path/path.dart' as path;
 
+import '../../test/mocks.dart';
 import 'rules/no_solo_tests.dart';
 import 'rules/visit_registered_nodes.dart';
 
 Future<void> main() async {
+  var results = await runChecks();
+  if (results.isNotEmpty) {
+    io.exitCode = 3;
+  }
+}
+
+var customChecks = [VisitRegisteredNodes(), NoSoloTests()];
+
+Future<List<AnalysisError>> runChecks() async {
   var rules =
       path.normalize(io.File(path.join('lib', 'src', 'rules')).absolute.path);
   var tests = path.normalize(io.File(path.join('test')).absolute.path);
-  await Driver([VisitRegisteredNodes(), NoSoloTests()]).analyze([rules, tests]);
+  var results = await Driver(customChecks).analyze([rules, tests]);
+  return results;
 }
 
 class Driver {
   Logger logger = Logger.standard();
 
-  List<LintRule> lints;
-  bool silent = false;
-
-  Driver(this.lints);
+  final List<LintRule> lints;
+  final bool silent;
 
-  Future analyze(List<String> sources, {bool displayTiming = false}) {
-    var analysisFuture = _analyze(sources);
-    if (!displayTiming) return analysisFuture;
+  Driver(this.lints, {this.silent = true});
 
-    var stopwatch = Stopwatch()..start();
-    return analysisFuture.then((value) {
-      _print(
-          '(Elapsed time: ${Duration(milliseconds: stopwatch.elapsedMilliseconds)})');
-    });
-  }
-
-  Future _analyze(List<String> sources) async {
+  Future<List<AnalysisError>> analyze(List<String> sources) async {
     if (sources.isEmpty) {
       _print('Specify one or more files and directories.');
-      return;
+      return [];
     }
     ResourceProvider resourceProvider = PhysicalResourceProvider.INSTANCE;
-    await _analyzeFiles(resourceProvider, sources);
+    var failedChecks = await _analyzeFiles(resourceProvider, sources);
     _print('Finished.');
+    return failedChecks;
   }
 
-  Future _analyzeFiles(
+  Future<List<AnalysisError>> _analyzeFiles(
       ResourceProvider resourceProvider, List<String> analysisRoots) async {
     _print('Analyzing...');
 
     // Register our checks.
     lints.forEach(Registry.ruleRegistry.register);
 
+    // Track failures.
+    var failedChecks = <AnalysisError>{};
+
     for (var root in analysisRoots) {
       var collection = AnalysisContextCollection(
         includedPaths: [root],
@@ -98,12 +103,16 @@ class Driver {
           }
         }
       }
-      ReportFormatter(errors, null /*_TodoFilter()*/, io.stdout).write();
+      ReportFormatter(
+              errors, null /*_TodoFilter()*/, silent ? MockIOSink() : io.stdout)
+          .write();
 
-      if (errors.isNotEmpty) {
-        io.exitCode = 3;
+      for (var info in errors) {
+        failedChecks.addAll(info.errors);
       }
     }
+
+    return failedChecks.toList();
   }
 
   /// Pass the following [msg] to the [logger] instance iff [silent] is false.