test_runner: add reporters

Author: MoLow

lib/internal/main/test_runner.js

@@ -6,6 +6,7 @@ const {
 const { getOptionValue } = require('internal/options');
 const { isUsingInspector } = require('internal/util/inspector');
 const { run } = require('internal/test_runner/runner');
+const { getTestReporter } = require('internal/test_runner/utils');
 const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 
 prepareMainThreadExecution(false);
@@ -21,7 +22,7 @@ if (isUsingInspector()) {
   inspectPort = process.debugPort;
 }
 
-const tapStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') });
+const tapStream = getTestReporter(run({ concurrency, inspectPort, watch: getOptionValue('--watch') }));
 tapStream.pipe(process.stdout);
 tapStream.once('test:fail', () => {
   process.exitCode = kGenericUserError;

lib/internal/test_runner/harness.js

@@ -18,6 +18,7 @@ const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { kEmptyObject } = require('internal/util');
 const { getOptionValue } = require('internal/options');
 const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test');
+const { getTestReporter } = require('internal/test_runner/utils');
 const { bigint: hrtime } = process.hrtime;
 
 const isTestRunnerCli = getOptionValue('--test');
@@ -119,8 +120,9 @@ let globalRoot;
 function getGlobalRoot() {
   if (!globalRoot) {
     globalRoot = createTestTree();
-    globalRoot.reporter.pipe(process.stdout);
-    globalRoot.reporter.once('test:fail', () => {
+    const reporter = getTestReporter(globalRoot.reporter);
+    reporter.pipe(process.stdout);
+    reporter.once('test:fail', () => {
       process.exitCode = kGenericUserError;
     });
   }

lib/internal/test_runner/runner.js

@@ -33,8 +33,9 @@ const { validateArray, validateBoolean } = require('internal/validators');
 const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { kEmptyObject } = require('internal/util');
 const { createTestTree } = require('internal/test_runner/harness');
-const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
+const { kSubtestsFailed, Test } = require('internal/test_runner/test');
 const { TapParser } = require('internal/test_runner/tap_parser');
+const { kDefaultIndent } = require('internal/test_runner/tap_stream');
 const { TokenKind } = require('internal/test_runner/tap_lexer');
 
 const {
@@ -49,6 +50,7 @@ const {
 } = internalBinding('errors');
 
 const kFilterArgs = ['--test', '--watch'];
+const kFilterArgValues = ['--test-reporter'];
 
 // TODO(cjihrig): Replace this with recursive readdir once it lands.
 function processPath(path, testFiles, options) {
@@ -112,8 +114,10 @@ function createTestFileList() {
   return ArrayPrototypeSort(ArrayFrom(testFiles));
 }
 
-function filterExecArgv(arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+function filterExecArgv(arg, i, arr) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg) &&
+    !ArrayPrototypeIncludes(kFilterArgValues, arg) &&
+    !ArrayPrototypeIncludes(kFilterArgValues, arr[i - 1]);
 }
 
 function getRunArgs({ path, inspectPort }) {
@@ -128,9 +132,10 @@ function getRunArgs({ path, inspectPort }) {
 class FileTest extends Test {
   #buffer = [];
   #handleReportItem({ kind, node, nesting = 0 }) {
-    const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
+    nesting += 1;
 
     const details = (diagnostic) => {
+      const indent = StringPrototypeRepeat(kDefaultIndent, nesting);
       return (
         diagnostic && {
           __proto__: null,
@@ -149,11 +154,11 @@ class FileTest extends Test {
         break;
 
       case TokenKind.TAP_PLAN:
-        this.reporter.plan(indent, node.end - node.start + 1);
+        this.reporter.plan(nesting, node.end - node.start + 1);
         break;
 
       case TokenKind.TAP_SUBTEST_POINT:
-        this.reporter.subtest(indent, node.name);
+        this.reporter.subtest(nesting, node.name);
         break;
 
       case TokenKind.TAP_TEST_POINT:
@@ -172,15 +177,15 @@ class FileTest extends Test {
 
         if (pass) {
           this.reporter.ok(
-            indent,
+            nesting,
             node.id,
             node.description,
             details(node.diagnostics),
             directive
           );
         } else {
           this.reporter.fail(
-            indent,
+            nesting,
             node.id,
             node.description,
             details(node.diagnostics),
@@ -190,15 +195,15 @@ class FileTest extends Test {
         break;
 
       case TokenKind.COMMENT:
-        if (indent === kDefaultIndent) {
+        if (nesting === 1) {
           // Ignore file top level diagnostics
           break;
         }
-        this.reporter.diagnostic(indent, node.comment);
+        this.reporter.diagnostic(nesting, node.comment);
         break;
 
       case TokenKind.UNKNOWN:
-        this.reporter.diagnostic(indent, node.value);
+        this.reporter.diagnostic(nesting, node.value);
         break;
     }
   }

lib/internal/test_runner/tap_stream.js

@@ -9,11 +9,13 @@ const {
   StringPrototypeReplaceAll,
   StringPrototypeToUpperCase,
   StringPrototypeSplit,
+  StringPrototypeRepeat,
   RegExpPrototypeSymbolReplace,
 } = primordials;
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const Readable = require('internal/streams/readable');
 const { isError, kEmptyObject } = require('internal/util');
+const kDefaultIndent = '    '; // 4 spaces
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
 const kDefaultTAPVersion = 13;
@@ -30,8 +32,8 @@ class TapStream extends Readable {
   #buffer;
   #canPush;
 
-  constructor() {
-    super();
+  constructor(options = kEmptyObject) {
+    super(options);
     this.#buffer = [];
     this.#canPush = true;
   }
@@ -40,34 +42,30 @@ class TapStream extends Readable {
     this.#canPush = true;
 
     while (this.#buffer.length > 0) {
-      const line = ArrayPrototypeShift(this.#buffer);
+      const chunk = ArrayPrototypeShift(this.#buffer);
 
-      if (!this.#tryPush(line)) {
+      if (!this.#tryPush(chunk)) {
         return;
       }
     }
   }
 
-  bail(message) {
-    this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`);
+  fail(nesting, testNumber, name, details, directive) {
+    this.#emit('test:fail', { __proto__: null, name, nesting, testNumber, details, ...directive });
+    this.#test(nesting, testNumber, 'not ok', name, directive);
+    this.#details(nesting, details);
   }
 
-  fail(indent, testNumber, name, details, directive) {
-    this.emit('test:fail', { __proto__: null, name, testNumber, details, ...directive });
-    this.#test(indent, testNumber, 'not ok', name, directive);
-    this.#details(indent, details);
+  ok(nesting, testNumber, name, details, directive) {
+    this.#emit('test:pass', { __proto__: null, name, nesting, testNumber, details, ...directive });
+    this.#test(nesting, testNumber, 'ok', name, directive);
+    this.#details(nesting, details);
   }
 
-  ok(indent, testNumber, name, details, directive) {
-    this.emit('test:pass', { __proto__: null, name, testNumber, details, ...directive });
-    this.#test(indent, testNumber, 'ok', name, directive);
-    this.#details(indent, details);
-  }
-
-  plan(indent, count, explanation) {
+  plan(nesting, count, explanation) {
     const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`;
 
-    this.#tryPush(`${indent}1..${count}${exp}\n`);
+    this.#tryPushString(`${this.#indent(nesting)}1..${count}${exp}\n`);
   }
 
   getSkip(reason) {
@@ -78,32 +76,42 @@ class TapStream extends Readable {
     return { __proto__: null, todo: reason };
   }
 
-  subtest(indent, name) {
-    this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`);
+  subtest(nesting, name) {
+    this.#emit('test:subtest', { nesting, name });
+    this.#tryPushString(`${this.#indent(nesting)}# Subtest: ${tapEscape(name)}\n`);
   }
 
-  #details(indent, data = kEmptyObject) {
+  #details(nesting, data = kEmptyObject) {
     const { error, duration, yaml } = data;
+    const indent = this.#indent(nesting);
     let details = `${indent}  ---\n`;
 
     details += `${yaml ? yaml : ''}`;
     details += jsToYaml(indent, 'duration_ms', duration);
     details += jsToYaml(indent, null, error);
     details += `${indent}  ...\n`;
-    this.#tryPush(details);
+    this.#tryPushString(details);
   }
 
-  diagnostic(indent, message) {
-    this.emit('test:diagnostic', message);
-    this.#tryPush(`${indent}# ${tapEscape(message)}\n`);
+  diagnostic(nesting, message) {
+    this.#emit('test:diagnostic', message);
+    this.#tryPushString(`${this.#indent(nesting)}# ${tapEscape(message)}\n`);
   }
 
   version(spec = kDefaultTAPVersion) {
-    this.#tryPush(`TAP version ${spec}\n`);
+    this.#tryPushString(`TAP version ${spec}\n`);
+  }
+
+  #indent(nesting) {
+    return StringPrototypeRepeat(kDefaultIndent, nesting);
   }
 
-  #test(indent, testNumber, status, name, directive = kEmptyObject) {
-    let line = `${indent}${status} ${testNumber}`;
+  #test(nesting, testNumber, status, name, directive = kEmptyObject) {
+    if (this._readableState.objectMode) {
+      // early return
+      return;
+    }
+    let line = `${this.#indent(nesting)}${status} ${testNumber}`;
 
     if (name) {
       line += ` ${tapEscape(`- ${name}`)}`;
@@ -115,7 +123,23 @@ class TapStream extends Readable {
 
     line += '\n';
 
-    this.#tryPush(line);
+    this.#tryPushString(line);
+  }
+
+  #emit(type, data) {
+    this.emit(type, data);
+    this.#tryPushObject({ type, data });
+  }
+
+  #tryPushString(str) {
+    if (!this._readableState.objectMode) {
+      this.#tryPush(str);
+    }
+  }
+  #tryPushObject(obj) {
+    if (this._readableState.objectMode) {
+      this.#tryPush(obj);
+    }
   }
 
   #tryPush(message) {
@@ -261,4 +285,4 @@ function isAssertionLike(value) {
   return value && typeof value === 'object' && 'expected' in value && 'actual' in value;
 }
 
-module.exports = { TapStream };
+module.exports = { TapStream, kDefaultIndent };

lib/internal/test_runner/test.js

@@ -63,9 +63,9 @@ const kTestCodeFailure = 'testCodeFailure';
 const kTestTimeoutFailure = 'testTimeoutFailure';
 const kHookFailure = 'hookFailed';
 const kDefaultTimeout = null;
-const kDefaultIndent = '    '; // 4 spaces
 const noop = FunctionPrototype;
 const isTestRunner = getOptionValue('--test');
+const hasReporters = Boolean(getOptionValue('--test-reporter'));
 const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
 const testNamePatternFlag = isTestRunner ? null :
   getOptionValue('--test-name-pattern');
@@ -171,18 +171,18 @@ class Test extends AsyncResource {
 
     if (parent === null) {
       this.concurrency = 1;
-      this.indent = '';
+      this.nesting = 0;
       this.only = testOnlyFlag;
-      this.reporter = new TapStream();
+      this.reporter = new TapStream({ objectMode: hasReporters });
       this.runOnlySubtests = this.only;
       this.testNumber = 0;
       this.timeout = kDefaultTimeout;
     } else {
-      const indent = parent.parent === null ? parent.indent :
-        parent.indent + kDefaultIndent;
+      const nesting = parent.parent === null ? parent.nesting :
+        parent.nesting + 1;
 
       this.concurrency = parent.concurrency;
-      this.indent = indent;
+      this.nesting = nesting;
       this.only = only ?? !parent.runOnlySubtests;
       this.reporter = parent.reporter;
       this.runOnlySubtests = !this.only;
@@ -607,19 +607,19 @@ class Test extends AsyncResource {
       this.parent.processPendingSubtests();
     } else if (!this.reported) {
       this.reported = true;
-      this.reporter.plan(this.indent, this.subtests.length);
+      this.reporter.plan(this.nesting, this.subtests.length);
 
       for (let i = 0; i < this.diagnostics.length; i++) {
-        this.reporter.diagnostic(this.indent, this.diagnostics[i]);
+        this.reporter.diagnostic(this.nesting, this.diagnostics[i]);
       }
 
-      this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`);
-      this.reporter.diagnostic(this.indent, `pass ${counters.passed}`);
-      this.reporter.diagnostic(this.indent, `fail ${counters.failed}`);
-      this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`);
-      this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`);
-      this.reporter.diagnostic(this.indent, `todo ${counters.todo}`);
-      this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`);
+      this.reporter.diagnostic(this.nesting, `tests ${this.subtests.length}`);
+      this.reporter.diagnostic(this.nesting, `pass ${counters.passed}`);
+      this.reporter.diagnostic(this.nesting, `fail ${counters.failed}`);
+      this.reporter.diagnostic(this.nesting, `cancelled ${counters.cancelled}`);
+      this.reporter.diagnostic(this.nesting, `skipped ${counters.skipped}`);
+      this.reporter.diagnostic(this.nesting, `todo ${counters.todo}`);
+      this.reporter.diagnostic(this.nesting, `duration_ms ${this.#duration()}`);
       this.reporter.push(null);
     }
   }
@@ -655,7 +655,7 @@ class Test extends AsyncResource {
 
   report() {
     if (this.subtests.length > 0) {
-      this.reporter.plan(this.subtests[0].indent, this.subtests.length);
+      this.reporter.plan(this.subtests[0].nesting, this.subtests.length);
     } else {
       this.reportSubtest();
     }
@@ -669,14 +669,14 @@ class Test extends AsyncResource {
     }
 
     if (this.passed) {
-      this.reporter.ok(this.indent, this.testNumber, this.name, details, directive);
+      this.reporter.ok(this.nesting, this.testNumber, this.name, details, directive);
     } else {
       details.error = this.error;
-      this.reporter.fail(this.indent, this.testNumber, this.name, details, directive);
+      this.reporter.fail(this.nesting, this.testNumber, this.name, details, directive);
     }
 
     for (let i = 0; i < this.diagnostics.length; i++) {
-      this.reporter.diagnostic(this.indent, this.diagnostics[i]);
+      this.reporter.diagnostic(this.nesting, this.diagnostics[i]);
     }
   }
 
@@ -686,7 +686,7 @@ class Test extends AsyncResource {
     }
     this.#reportedSubtest = true;
     this.parent.reportSubtest();
-    this.reporter.subtest(this.indent, this.name);
+    this.reporter.subtest(this.nesting, this.name);
   }
 }
 
@@ -790,7 +790,6 @@ class Suite extends Test {
 module.exports = {
   ItTest,
   kCancelledByParent,
-  kDefaultIndent,
   kSubtestsFailed,
   kTestCodeFailure,
   kUnwrapErrors,