test_runner: report coverage thresholds in test:coverage

PR-URL: #54813
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>

Author: avivkeller

doc/api/test.md

@@ -2806,6 +2806,11 @@ are defined, while others are emitted in the order that the tests execute.
         numbers and the number of times they were covered.
         * `line` {number} The line number.
         * `count` {number} The number of times the line was covered.
+    * `thresholds` {Object} An object containing whether or not the coverage for
+      each coverage type.
+      * `function` {number} The function coverage threshold.
+      * `branch` {number} The branch coverage threshold.
+      * `line` {number} The line coverage threshold.
     * `totals` {Object} An object containing a summary of coverage for all
       files.
       * `totalLineCount` {number} The total number of lines.

lib/internal/test_runner/coverage.js

@@ -51,12 +51,13 @@ class CoverageLine {
 }
 
 class TestCoverage {
-  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs) {
+  constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs, thresholds) {
     this.coverageDirectory = coverageDirectory;
     this.originalCoverageDirectory = originalCoverageDirectory;
     this.workingDirectory = workingDirectory;
     this.excludeGlobs = excludeGlobs;
     this.includeGlobs = includeGlobs;
+    this.thresholds = thresholds;
   }
 
   #sourceLines = new SafeMap();
@@ -143,6 +144,7 @@ class TestCoverage {
         coveredBranchPercent: 0,
         coveredFunctionPercent: 0,
       },
+      thresholds: this.thresholds,
     };
 
     if (!coverage) {
@@ -505,6 +507,12 @@ function setupCoverage(options) {
     cwd,
     options.coverageExcludeGlobs,
     options.coverageIncludeGlobs,
+    {
+      __proto__: null,
+      line: options.lineCoverage,
+      branch: options.branchCoverage,
+      function: options.functionCoverage,
+    },
   );
 }
 

lib/internal/test_runner/test.js

@@ -1043,8 +1043,6 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
-        reporter.coverage(nesting, loc, coverage);
-
         const coverages = [
           { __proto__: null, actual: coverage.totals.coveredLinePercent,
             threshold: this.config.lineCoverage, name: 'line' },
@@ -1063,6 +1061,8 @@ class Test extends AsyncResource {
             reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
           }
         }
+
+        reporter.coverage(nesting, loc, coverage);
       }
 
       if (harness.watching) {

test/parallel/test-runner-coverage-thresholds.js

@@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() {
 }
 
 const fixture = fixtures.path('test-runner', 'coverage.js');
-const neededArguments = [
-  '--experimental-test-coverage',
-  '--test-reporter', 'tap',
-];
+const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs');
 
 const coverages = [
   { flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
@@ -56,10 +53,12 @@ const coverages = [
 ];
 
 for (const coverage of coverages) {
-  test(`test passing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=25`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -70,10 +69,27 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test failing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=25`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert.strictEqual(stdout.summary.thresholds[coverage.name], 25);
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test failing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=99`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -84,9 +100,25 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
+  test(`test failing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=99`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert.strictEqual(stdout.summary.thresholds[coverage.name], 99);
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test out-of-range ${coverage.flag} (too high)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=101`,
       fixture,
     ]);
@@ -96,9 +128,10 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
+  test(`test out-of-range ${coverage.flag} (too low)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=-1`,
       fixture,
     ]);