From 7724aabe8cd40b376573330f5227a6ec0a47f28e Mon Sep 17 00:00:00 2001 From: Nate Bosch Date: Mon, 4 Mar 2024 11:00:11 -0800 Subject: [PATCH] Add failure only reporter (#2190) Closes #829 Copies the expanded reporter and test and removes output unrelated to failed tests. --- pkgs/test/pubspec.yaml | 2 +- .../runner/failures_only_reporter_test.dart | 257 +++++++++++++++ pkgs/test/test/runner/runner_test.dart | 1 + .../src/runner/configuration/reporters.dart | 9 + .../src/runner/reporter/failures_only.dart | 295 ++++++++++++++++++ 5 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 pkgs/test/test/runner/failures_only_reporter_test.dart create mode 100644 pkgs/test_core/lib/src/runner/reporter/failures_only.dart diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index 531fb20d2..64394819b 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml @@ -35,7 +35,7 @@ dependencies: # Use an exact version until the test_api and test_core package are stable. test_api: 0.7.0 - test_core: 0.6.0 + test_core: 0.6.1 typed_data: ^1.3.0 web_socket_channel: ^2.0.0 diff --git a/pkgs/test/test/runner/failures_only_reporter_test.dart b/pkgs/test/test/runner/failures_only_reporter_test.dart new file mode 100644 index 000000000..3e2690abc --- /dev/null +++ b/pkgs/test/test/runner/failures_only_reporter_test.dart @@ -0,0 +1,257 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm') +library; + +import 'dart:async'; + +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; + +import '../io.dart'; + +void main() { + setUpAll(precompileTestExecutable); + + test('reports when no tests are run', () async { + await d.file('test.dart', 'void main() {}').create(); + + var test = await runTest(['test.dart'], reporter: 'failures-only'); + expect(test.stdout, emitsThrough(contains('No tests ran.'))); + await test.shouldExit(79); + }); + + test('runs several successful tests and reports only at the end', () { + return _expectReport(''' + test('success 1', () {}); + test('success 2', () {}); + test('success 3', () {});''', ''' + +3: All tests passed!'''); + }); + + test('runs several failing tests and reports when each fails', () { + return _expectReport(''' + test('failure 1', () => throw TestFailure('oh no')); + test('failure 2', () => throw TestFailure('oh no')); + test('failure 3', () => throw TestFailure('oh no'));''', ''' + +0 -1: failure 1 [E] + oh no + test.dart 6:33 main. + + +0 -2: failure 2 [E] + oh no + test.dart 7:33 main. + + +0 -3: failure 3 [E] + oh no + test.dart 8:33 main. + + +0 -3: Some tests failed.'''); + }); + + test('includes the full stack trace with --verbose-trace', () async { + await d.file('test.dart', ''' +import 'dart:async'; + +import 'package:test/test.dart'; + +void main() { + test("failure", () => throw "oh no"); +} +''').create(); + + var test = await runTest(['--verbose-trace', 'test.dart'], + reporter: 'failures-only'); + expect(test.stdout, emitsThrough(contains('dart:async'))); + await test.shouldExit(1); + }); + + test('reports only failing tests amid successful tests', () { + return _expectReport(''' + test('failure 1', () => throw TestFailure('oh no')); + test('success 1', () {}); + test('failure 2', () => throw TestFailure('oh no')); + test('success 2', () {});''', ''' + +0 -1: failure 1 [E] + oh no + test.dart 6:33 main. + + +1 -2: failure 2 [E] + oh no + test.dart 8:33 main. + + +2 -2: Some tests failed.'''); + }); + + group('print:', () { + test('handles multiple prints', () { + return _expectReport(''' + test('test', () { + print("one"); + print("two"); + print("three"); + print("four"); + });''', ''' + +0: test + one + two + three + four + +1: All tests passed!'''); + }); + + test('handles a print after the test completes', () { + return _expectReport(''' + // This completer ensures that the test isolate isn't killed until all + // prints have happened. + var testDone = Completer(); + var waitStarted = Completer(); + test('test', () async { + waitStarted.future.then((_) { + Future(() => print("one")); + Future(() => print("two")); + Future(() => print("three")); + Future(() => print("four")); + Future(testDone.complete); + }); + }); + + test('wait', () { + waitStarted.complete(); + return testDone.future; + });''', ''' + +1: test + one + two + three + four + +2: All tests passed!'''); + }); + + test('interleaves prints and errors', () { + return _expectReport(''' + // This completer ensures that the test isolate isn't killed until all + // prints have happened. + var completer = Completer(); + test('test', () { + scheduleMicrotask(() { + print("three"); + print("four"); + throw "second error"; + }); + + scheduleMicrotask(() { + print("five"); + print("six"); + completer.complete(); + }); + + print("one"); + print("two"); + throw "first error"; + }); + + test('wait', () => completer.future);''', ''' + +0: test + one + two + +0 -1: test [E] + first error + test.dart 24:11 main. + + three + four + second error + test.dart 13:13 main.. + ===== asynchronous gap =========================== + dart:async scheduleMicrotask + test.dart 10:11 main. + + five + six + +1 -1: Some tests failed.'''); + }); + }); + + group('skip:', () { + test('does not emit for skips', () { + return _expectReport(''' + test('skip 1', () {}, skip: true); + test('skip 2', () {}, skip: true); + test('skip 3', () {}, skip: true);''', ''' + +0 ~3: All tests skipped.'''); + }); + + test('runs skipped tests along with successful and failing tests', () { + return _expectReport(''' + test('failure 1', () => throw TestFailure('oh no')); + test('skip 1', () {}, skip: true); + test('success 1', () {}); + test('failure 2', () => throw TestFailure('oh no')); + test('skip 2', () {}, skip: true); + test('success 2', () {});''', ''' + +0 -1: failure 1 [E] + oh no + test.dart 6:35 main. + + +1 ~1 -2: failure 2 [E] + oh no + test.dart 9:35 main. + + +2 ~2 -2: Some tests failed.'''); + }); + }); + + test('Directs users to enable stack trace chaining if disabled', () async { + await _expectReport( + '''test('failure 1', () => throw TestFailure('oh no'));''', ''' + +0 -1: failure 1 [E] + oh no + test.dart 6:25 main. + + +0 -1: Some tests failed. + + Consider enabling the flag chain-stack-traces to receive more detailed exceptions. + For example, 'dart test --chain-stack-traces'.''', + chainStackTraces: false); + }); +} + +Future _expectReport(String tests, String expected, + {List args = const [], bool chainStackTraces = true}) async { + await d.file('test.dart', ''' + import 'dart:async'; + + import 'package:test/test.dart'; + + void main() { +$tests + } + ''').create(); + + var test = await runTest([ + 'test.dart', + if (chainStackTraces) '--chain-stack-traces', + ...args, + ], reporter: 'failures-only'); + await test.shouldExit(); + + var stdoutLines = await test.stdoutStream().toList(); + + // Remove excess trailing whitespace. + var actual = stdoutLines.map((line) { + if (line.startsWith(' ') || line.isEmpty) return line.trimRight(); + return line.trim(); + }).join('\n'); + + // Un-indent the expected string. + var indentation = expected.indexOf(RegExp('[^ ]')); + expected = expected.split('\n').map((line) { + if (line.isEmpty) return line; + return line.substring(indentation); + }).join('\n'); + + expect(actual, equals(expected)); +} diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index dde3d049e..c16ac517b 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -108,6 +108,7 @@ Output: [compact] A single line, updated continuously. [expanded] (default) A separate line for each update. + [failures-only] A separate line for failing tests with no output for passing tests [github] A custom reporter for GitHub Actions (the default reporter when running on GitHub Actions). [json] A machine-readable format (see https://dart.dev/go/test-docs/json_reporter.md). [silent] A reporter with no output. May be useful when only the exit code is meaningful. diff --git a/pkgs/test_core/lib/src/runner/configuration/reporters.dart b/pkgs/test_core/lib/src/runner/configuration/reporters.dart index a79912f87..0f0fe0176 100644 --- a/pkgs/test_core/lib/src/runner/configuration/reporters.dart +++ b/pkgs/test_core/lib/src/runner/configuration/reporters.dart @@ -11,6 +11,7 @@ import '../engine.dart'; import '../reporter.dart'; import '../reporter/compact.dart'; import '../reporter/expanded.dart'; +import '../reporter/failures_only.dart'; import '../reporter/github.dart'; import '../reporter/json.dart'; @@ -46,6 +47,14 @@ final _allReporters = { Directory(config.testSelections.keys.single).existsSync(), printPlatform: config.suiteDefaults.runtimes.length > 1 || config.suiteDefaults.compilerSelections != null)), + 'failures-only': ReporterDetails( + 'A separate line for failing tests with no output for passing tests', + (config, engine, sink) => FailuresOnlyReporter.watch(engine, sink, + color: config.color, + printPath: config.testSelections.length > 1 || + Directory(config.testSelections.keys.single).existsSync(), + printPlatform: config.suiteDefaults.runtimes.length > 1 || + config.suiteDefaults.compilerSelections != null)), 'github': ReporterDetails( 'A custom reporter for GitHub Actions ' '(the default reporter when running on GitHub Actions).', diff --git a/pkgs/test_core/lib/src/runner/reporter/failures_only.dart b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart new file mode 100644 index 000000000..b6b85e7d1 --- /dev/null +++ b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart @@ -0,0 +1,295 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; + +import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/message.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports + +import '../../util/pretty_print.dart'; +import '../engine.dart'; +import '../load_exception.dart'; +import '../load_suite.dart'; +import '../reporter.dart'; + +/// A reporter that only prints when a test fails. +class FailuresOnlyReporter implements Reporter { + /// Whether the reporter should emit terminal color escapes. + final bool _color; + + /// The terminal escape for green text, or the empty string if this is Windows + /// or not outputting to a terminal. + final String _green; + + /// The terminal escape for red text, or the empty string if this is Windows + /// or not outputting to a terminal. + final String _red; + + /// The terminal escape for yellow text, or the empty string if this is + /// Windows or not outputting to a terminal. + final String _yellow; + + /// The terminal escape for gray text, or the empty string if this is + /// Windows or not outputting to a terminal. + final String _gray; + + /// The terminal escape for bold text, or the empty string if this is + /// Windows or not outputting to a terminal. + final String _bold; + + /// The terminal escape for removing test coloring, or the empty string if + /// this is Windows or not outputting to a terminal. + final String _noColor; + + /// The engine used to run the tests. + final Engine _engine; + + /// Whether the path to each test's suite should be printed. + final bool _printPath; + + /// Whether the platform each test is running on should be printed. + final bool _printPlatform; + + /// The size of `_engine.passed` last time a progress notification was + /// printed. + int _lastProgressPassed = 0; + + /// The size of `_engine.skipped` last time a progress notification was + /// printed. + int _lastProgressSkipped = 0; + + /// The size of `_engine.failed` last time a progress notification was + /// printed. + int _lastProgressFailed = 0; + + /// The message printed for the last progress notification. + String _lastProgressMessage = ''; + + /// The suffix added to the last progress notification. + String? _lastProgressSuffix; + + /// Whether the reporter is paused. + var _paused = false; + + // Whether a notice should be logged about enabling stack trace chaining at + // the end of all tests running. + var _shouldPrintStackTraceChainingNotice = false; + + /// The set of all subscriptions to various streams. + final _subscriptions = {}; + + final StringSink _sink; + + /// Watches the tests run by [engine] and prints their results to the + /// terminal. + /// + /// If [color] is `true`, this will use terminal colors; if it's `false`, it + /// won't. If [printPath] is `true`, this will print the path name as part of + /// the test description. Likewise, if [printPlatform] is `true`, this will + /// print the platform as part of the test description. + static FailuresOnlyReporter watch(Engine engine, StringSink sink, + {required bool color, + required bool printPath, + required bool printPlatform}) => + FailuresOnlyReporter._(engine, sink, + color: color, printPath: printPath, printPlatform: printPlatform); + + FailuresOnlyReporter._(this._engine, this._sink, + {required bool color, + required bool printPath, + required bool printPlatform}) + : _printPath = printPath, + _printPlatform = printPlatform, + _color = color, + _green = color ? '\u001b[32m' : '', + _red = color ? '\u001b[31m' : '', + _yellow = color ? '\u001b[33m' : '', + _gray = color ? '\u001b[90m' : '', + _bold = color ? '\u001b[1m' : '', + _noColor = color ? '\u001b[0m' : '' { + _subscriptions.add(_engine.onTestStarted.listen(_onTestStarted)); + + // Convert the future to a stream so that the subscription can be paused or + // canceled. + _subscriptions.add(_engine.success.asStream().listen(_onDone)); + } + + @override + void pause() { + if (_paused) return; + _paused = true; + + for (var subscription in _subscriptions) { + subscription.pause(); + } + } + + @override + void resume() { + if (!_paused) return; + + for (var subscription in _subscriptions) { + subscription.resume(); + } + } + + void _cancel() { + for (var subscription in _subscriptions) { + subscription.cancel(); + } + _subscriptions.clear(); + } + + /// A callback called when the engine begins running [liveTest]. + void _onTestStarted(LiveTest liveTest) { + _subscriptions.add(liveTest.onError + .listen((error) => _onError(liveTest, error.error, error.stackTrace))); + + _subscriptions.add(liveTest.onMessage.listen((message) { + // TODO - Should this suppress output? Behave like printOnFailure? + _progressLine(_description(liveTest)); + var text = message.text; + if (message.type == MessageType.skip) text = ' $_yellow$text$_noColor'; + _sink.writeln(text); + })); + } + + /// A callback called when [liveTest] throws [error]. + void _onError(LiveTest liveTest, Object error, StackTrace stackTrace) { + if (!liveTest.test.metadata.chainStackTraces && + !liveTest.suite.isLoadSuite) { + _shouldPrintStackTraceChainingNotice = true; + } + + if (liveTest.state.status != Status.complete) return; + + _progressLine(_description(liveTest), suffix: ' $_bold$_red[E]$_noColor'); + + if (error is! LoadException) { + _sink + ..writeln(indent('$error')) + ..writeln(indent('$stackTrace')); + return; + } + + // TODO - what type is this? + _sink.writeln(indent(error.toString(color: _color))); + + // Only print stack traces for load errors that come from the user's code. + if (error.innerError is! FormatException && error.innerError is! String) { + _sink.writeln(indent('$stackTrace')); + } + } + + /// A callback called when the engine is finished running tests. + /// + /// [success] will be `true` if all tests passed, `false` if some tests + /// failed, and `null` if the engine was closed prematurely. + void _onDone(bool? success) { + _cancel(); + // A null success value indicates that the engine was closed before the + // tests finished running, probably because of a signal from the user, in + // which case we shouldn't print summary information. + if (success == null) return; + + if (_engine.liveTests.isEmpty) { + _sink.writeln('No tests ran.'); + } else if (!success) { + for (var liveTest in _engine.active) { + _progressLine(_description(liveTest), + suffix: ' - did not complete $_bold$_red[E]$_noColor'); + } + _progressLine('Some tests failed.', color: _red); + } else if (_engine.passed.isEmpty) { + _progressLine('All tests skipped.'); + } else { + _progressLine('All tests passed!'); + } + + if (_shouldPrintStackTraceChainingNotice) { + _sink + ..writeln('') + ..writeln('Consider enabling the flag chain-stack-traces to ' + 'receive more detailed exceptions.\n' + "For example, 'dart test --chain-stack-traces'."); + } + } + + /// Prints a line representing the current state of the tests. + /// + /// [message] goes after the progress report. If [color] is passed, it's used + /// as the color for [message]. If [suffix] is passed, it's added to the end + /// of [message]. + void _progressLine(String message, {String? color, String? suffix}) { + // Print nothing if nothing has changed since the last progress line. + if (_engine.passed.length == _lastProgressPassed && + _engine.skipped.length == _lastProgressSkipped && + _engine.failed.length == _lastProgressFailed && + message == _lastProgressMessage && + // Don't re-print just because a suffix was removed. + (suffix == null || suffix == _lastProgressSuffix)) { + return; + } + + _lastProgressPassed = _engine.passed.length; + _lastProgressSkipped = _engine.skipped.length; + _lastProgressFailed = _engine.failed.length; + _lastProgressMessage = message; + _lastProgressSuffix = suffix; + + if (suffix != null) message += suffix; + color ??= ''; + var buffer = StringBuffer(); + + buffer.write(_green); + buffer.write('+'); + buffer.write(_engine.passed.length); + buffer.write(_noColor); + + if (_engine.skipped.isNotEmpty) { + buffer.write(_yellow); + buffer.write(' ~'); + buffer.write(_engine.skipped.length); + buffer.write(_noColor); + } + + if (_engine.failed.isNotEmpty) { + buffer.write(_red); + buffer.write(' -'); + buffer.write(_engine.failed.length); + buffer.write(_noColor); + } + + buffer.write(': '); + buffer.write(color); + buffer.write(message); + buffer.write(_noColor); + + _sink.writeln(buffer.toString()); + } + + /// Returns a description of [liveTest]. + /// + /// This differs from the test's own description in that it may also include + /// the suite's name. + String _description(LiveTest liveTest) { + var name = liveTest.test.name; + + if (_printPath && + liveTest.suite is! LoadSuite && + liveTest.suite.path != null) { + name = '${liveTest.suite.path}: $name'; + } + + if (_printPlatform) { + name = '[${liveTest.suite.platform.runtime.name}, ' + '${liveTest.suite.platform.compiler.name}] $name'; + } + + if (liveTest.suite is LoadSuite) name = '$_bold$_gray$name$_noColor'; + + return name; + } +}