test_runner: expose expectFailure message

This change exposes the expectFailure message in the test runner and
adds edge cases for expectFailure ambiguity.

PR-URL: #61563
Reviewed-By: vassudanagunta
Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

Author: Han5991

doc/api/test.md

@@ -265,11 +265,12 @@ added:
  - v25.5.0
 -->
 
-This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
-in order to "pass"; a test/test-case that does not throw, fails.
+This flips the pass/fail reporting for a specific test or suite: a flagged test
+case must throw in order to pass, and a flagged test case that does not throw
+fails.
 
-In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing
-`strictEqual` to throw, so the test-case passes).
+In each of the following, `doTheThing()` fails to return `true`, but since the
+tests are flagged `expectFailure`, they pass.
 
 ```js
 it.expectFailure('should do the thing', () => {
@@ -279,6 +280,50 @@ it.expectFailure('should do the thing', () => {
 it('should do the thing', { expectFailure: true }, () => {
   assert.strictEqual(doTheThing(), true);
 });
+
+it('should do the thing', { expectFailure: 'feature not implemented' }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+```
+
+If the value of `expectFailure` is a
+[<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp) |
+[<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function) |
+[<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) |
+[<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error),
+the tests will pass only if they throw a matching value.
+See [`assert.throws`][] for how each value type is handled.
+
+Each of the following tests fails _despite_ being flagged `expectFailure`
+because the failure does not match the specific **expected** failure.
+
+```js
+it('fails because regex does not match', {
+  expectFailure: /expected message/,
+}, () => {
+  throw new Error('different message');
+});
+
+it('fails because object matcher does not match', {
+  expectFailure: { code: 'ERR_EXPECTED' },
+}, () => {
+  const err = new Error('boom');
+  err.code = 'ERR_ACTUAL';
+  throw err;
+});
+```
+
+To supply both a reason and specific error for `expectFailure`, use `{ label, match }`.
+
+```js
+it('should fail with specific error and reason', {
+  expectFailure: {
+    label: 'reason for failure',
+    match: /error message/,
+  },
+}, () => {
+  assert.strictEqual(doTheThing(), true);
+});
 ```
 
 `skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
@@ -1684,6 +1729,18 @@ changes:
     thread. If `false`, only one test runs at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `false`.
+  * `expectFailure` {boolean|string|RegExp|Function|Object|Error} If truthy, the
+    test is expected to fail. If a non-empty string is provided, that string is displayed
+    in the test results as the reason why the test is expected to fail. If a
+    [<RegExp>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp),
+    [<Function>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function),
+    [<Object>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object), or
+    [<Error>](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error)
+    is provided directly (without wrapping in `{ match: … }`), the test passes
+    only if the thrown error matches, following the behavior of
+    [`assert.throws`][]. To provide both a reason and validation, pass an object
+    with `label` (string) and `match` (RegExp, Function, Object, or Error).
+    **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.
@@ -4122,6 +4179,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`NODE_V8_COVERAGE`]: cli.md#node_v8_coveragedir
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`assert.throws`]: assert.md#assertthrowsfn-error-message
 [`context.diagnostic`]: #contextdiagnosticmessage
 [`context.skip`]: #contextskipmessage
 [`context.todo`]: #contexttodomessage

lib/internal/test_runner/reporter/tap.js

@@ -77,7 +77,7 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
   } else if (todo !== undefined) {
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
-    line += ' # EXPECTED FAILURE';
+    line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' ? ` ${tapEscape(expectFailure)}` : ''}`;
   }
 
   line += '\n';

lib/internal/test_runner/test.js

@@ -1,5 +1,6 @@
 'use strict';
 const {
+  ArrayPrototypeEvery,
   ArrayPrototypePush,
   ArrayPrototypePushApply,
   ArrayPrototypeShift,
@@ -13,6 +14,7 @@ const {
   MathMax,
   Number,
   NumberPrototypeToFixed,
+  ObjectKeys,
   ObjectSeal,
   Promise,
   PromisePrototypeThen,
@@ -40,6 +42,7 @@ const {
   AbortError,
   codes: {
     ERR_INVALID_ARG_TYPE,
+    ERR_INVALID_ARG_VALUE,
     ERR_TEST_FAILURE,
   },
 } = require('internal/errors');
@@ -56,7 +59,8 @@ const {
   once: runOnce,
   setOwnProperty,
 } = require('internal/util');
-const { isPromise } = require('internal/util/types');
+const assert = require('assert');
+const { isPromise, isRegExp } = require('internal/util/types');
 const {
   validateAbortSignal,
   validateFunction,
@@ -487,6 +491,39 @@ class SuiteContext {
   }
 }
 
+function parseExpectFailure(expectFailure) {
+  if (expectFailure === undefined || expectFailure === false) {
+    return false;
+  }
+
+  if (typeof expectFailure === 'string') {
+    return { __proto__: null, label: expectFailure, match: undefined };
+  }
+
+  if (typeof expectFailure === 'function' || isRegExp(expectFailure)) {
+    return { __proto__: null, label: undefined, match: expectFailure };
+  }
+
+  if (typeof expectFailure !== 'object') {
+    return { __proto__: null, label: undefined, match: undefined };
+  }
+
+  const keys = ObjectKeys(expectFailure);
+  if (keys.length === 0) {
+    throw new ERR_INVALID_ARG_VALUE('options.expectFailure', expectFailure, 'must not be an empty object');
+  }
+
+  if (ArrayPrototypeEvery(keys, (k) => k === 'match' || k === 'label')) {
+    return {
+      __proto__: null,
+      label: expectFailure.label,
+      match: expectFailure.match,
+    };
+  }
+
+  return { __proto__: null, label: undefined, match: expectFailure };
+}
+
 class Test extends AsyncResource {
   reportedType = 'test';
   abortController;
@@ -636,7 +673,7 @@ class Test extends AsyncResource {
     this.plan = null;
     this.expectedAssertions = plan;
     this.cancelled = false;
-    this.expectFailure = expectFailure !== undefined && expectFailure !== false;
+    this.expectFailure = parseExpectFailure(expectFailure) || this.parent?.expectFailure;
     this.skipped = skip !== undefined && skip !== false;
     this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
     this.startTime = null;
@@ -950,7 +987,30 @@ class Test extends AsyncResource {
       return;
     }
 
-    if (this.expectFailure === true) {
+    if (this.expectFailure) {
+      if (typeof this.expectFailure === 'object' &&
+          this.expectFailure.match !== undefined) {
+        const { match: validation } = this.expectFailure;
+        try {
+          const errorToCheck = (
+            err?.code === 'ERR_TEST_FAILURE' &&
+            err?.failureType === kTestCodeFailure &&
+            err.cause
+          ) ?
+            err.cause :
+            err;
+          // eslint-disable-next-line no-restricted-syntax
+          assert.throws(() => { throw errorToCheck; }, validation);
+        } catch (e) {
+          this.passed = false;
+          this.error = new ERR_TEST_FAILURE(
+            'The test failed, but the error did not match the expected validation',
+            kTestCodeFailure,
+          );
+          this.error.cause = e;
+          return;
+        }
+      }
       this.passed = true;
     } else {
       this.passed = false;
@@ -960,7 +1020,7 @@ class Test extends AsyncResource {
   }
 
   pass() {
-    if (this.error == null && this.expectFailure === true && !this.skipped) {
+    if (this.error == null && this.expectFailure && !this.skipped) {
       this.passed = false;
       this.error = new ERR_TEST_FAILURE(
         'test was expected to fail but passed',
@@ -972,6 +1032,20 @@ class Test extends AsyncResource {
       return;
     }
 
+    if (this.skipped || this.isTodo) {
+      this.passed = true;
+      return;
+    }
+
+    if (this.expectFailure) {
+      this.passed = false;
+      this.error = new ERR_TEST_FAILURE(
+        'Test passed but was expected to fail',
+        kTestCodeFailure,
+      );
+      return;
+    }
+
     this.passed = true;
   }
 
@@ -1361,7 +1435,10 @@ class Test extends AsyncResource {
     } else if (this.isTodo) {
       directive = this.reporter.getTodo(this.message);
     } else if (this.expectFailure) {
-      directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
+      const message = typeof this.expectFailure === 'object' ?
+        this.expectFailure.label :
+        this.expectFailure;
+      directive = this.reporter.getXFail(message);
     }
 
     if (this.reportedType) {

test/parallel/test-runner-expect-error-but-pass.js

@@ -8,10 +8,9 @@ if (!process.env.NODE_TEST_CONTEXT) {
 
   stream.on('test:pass', common.mustNotCall());
   stream.on('test:fail', common.mustCall((event) => {
-    assert.strictEqual(event.expectFailure, true);
     assert.strictEqual(event.details.error.code, 'ERR_TEST_FAILURE');
     assert.strictEqual(event.details.error.failureType, 'expectedFailure');
-    assert.strictEqual(event.details.error.cause, 'test was expected to fail but passed');
+    assert.strictEqual(event.details.error.message, 'test was expected to fail but passed');
   }, 1));
 } else {
   test('passing test', { expectFailure: true }, () => {});