Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test_runner: add code coverage support to spec reporter #46674

Merged
merged 2 commits into from
Apr 2, 2023
Merged

test_runner: add code coverage support to spec reporter #46674

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca35e89
test_runner: add code coverage support to spec reporter
pulkit-30 Mar 26, 2023
1f1de83
added test-reporter explicitly
pulkit-30 Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion lib/internal/test_runner/reporter/spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const assert = require('assert');
const Transform = require('internal/streams/transform');
const { inspectWithNoCustomRetry } = require('internal/errors');
const { green, blue, red, white, gray } = require('internal/util/colors');

const { getCoverageReport } = require('internal/test_runner/utils');

const inspectOptions = { __proto__: null, colors: true, breakLength: Infinity };

Expand All @@ -30,6 +30,7 @@ const symbols = {
'test:fail': '\u2716 ',
'test:pass': '\u2714 ',
'test:diagnostic': '\u2139 ',
'test:coverage': '\u2139 ',
'arrow:right': '\u25B6 ',
'hyphen:minus': '\uFE63 ',
};
Expand Down Expand Up @@ -115,6 +116,8 @@ class SpecReporter extends Transform {
break;
case 'test:diagnostic':
return `${colors[type]}${this.#indent(data.nesting)}${symbols[type]}${data.message}${white}\n`;
case 'test:coverage':
return getCoverageReport(this.#indent(data.nesting), data.summary, symbols['test:coverage'], blue);
}
}
_transform({ type, data }, encoding, callback) {
Expand Down
39 changes: 2 additions & 37 deletions lib/internal/test_runner/reporter/tap.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ const {
ArrayPrototypeForEach,
ArrayPrototypeJoin,
ArrayPrototypePush,
NumberPrototypeToFixed,
ObjectEntries,
RegExpPrototypeSymbolReplace,
SafeMap,
Expand All @@ -13,7 +12,7 @@ const {
} = primordials;
const { inspectWithNoCustomRetry } = require('internal/errors');
const { isError, kEmptyObject } = require('internal/util');
const { relative } = require('path');
const { getCoverageReport } = require('internal/test_runner/utils');
const kDefaultIndent = ' '; // 4 spaces
const kFrameStartRegExp = /^ {4}at /;
const kLineBreakRegExp = /\n|\r\n/;
Expand Down Expand Up @@ -49,7 +48,7 @@ async function * tapReporter(source) {
yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`;
break;
case 'test:coverage':
yield reportCoverage(data.nesting, data.summary);
yield getCoverageReport(indent(data.nesting), data.summary, '# ', '');
break;
}
}
Expand All @@ -73,40 +72,6 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
return line;
}

function reportCoverage(nesting, summary) {
const pad = indent(nesting);
let report = `${pad}# start of coverage report\n`;

report += `${pad}# file | line % | branch % | funcs % | uncovered lines\n`;

for (let i = 0; i < summary.files.length; ++i) {
const {
path,
coveredLinePercent,
coveredBranchPercent,
coveredFunctionPercent,
uncoveredLineNumbers,
} = summary.files[i];
const relativePath = relative(summary.workingDirectory, path);
const lines = NumberPrototypeToFixed(coveredLinePercent, 2);
const branches = NumberPrototypeToFixed(coveredBranchPercent, 2);
const functions = NumberPrototypeToFixed(coveredFunctionPercent, 2);
const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');

report += `${pad}# ${relativePath} | ${lines} | ${branches} | ` +
`${functions} | ${uncovered}\n`;
}

const { totals } = summary;
report += `${pad}# all files | ` +
`${NumberPrototypeToFixed(totals.coveredLinePercent, 2)} | ` +
`${NumberPrototypeToFixed(totals.coveredBranchPercent, 2)} | ` +
`${NumberPrototypeToFixed(totals.coveredFunctionPercent, 2)} |\n`;

report += `${pad}# end of coverage report\n`;
return report;
}

function reportDetails(nesting, data = kEmptyObject) {
const { error, duration_ms } = data;
const _indent = indent(nesting);
Expand Down
52 changes: 51 additions & 1 deletion lib/internal/test_runner/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
'use strict';
const {
ArrayPrototypeJoin,
ArrayPrototypeMap,
ArrayPrototypePush,
ObjectGetOwnPropertyDescriptor,
NumberPrototypeToFixed,
SafePromiseAllReturnArrayLike,
RegExp,
RegExpPrototypeExec,
SafeMap,
} = primordials;

const { basename } = require('path');
const { basename, relative } = require('path');
const { createWriteStream } = require('fs');
const { pathToFileURL } = require('internal/url');
const { createDeferredPromise } = require('internal/util');
const { getOptionValue } = require('internal/options');
const { green, red, white } = require('internal/util/colors');

const {
codes: {
Expand Down Expand Up @@ -246,6 +249,52 @@ function countCompletedTest(test, harness = test.root.harness) {
harness.counters.all++;
}


function coverageThreshold(coverage, color) {
coverage = NumberPrototypeToFixed(coverage, 2);
if (color) {
if (coverage > 90) return `${green}${coverage}${color}`;
if (coverage < 50) return `${red}${coverage}${color}`;
}
return coverage;
}

function getCoverageReport(pad, summary, symbol, color) {
let report = `${color}${pad}${symbol}start of coverage report\n`;

report += `${pad}${symbol}file | line % | branch % | funcs % | uncovered lines\n`;

for (let i = 0; i < summary.files.length; ++i) {
const {
path,
coveredLinePercent,
coveredBranchPercent,
coveredFunctionPercent,
uncoveredLineNumbers,
} = summary.files[i];
const relativePath = relative(summary.workingDirectory, path);
const lines = coverageThreshold(coveredLinePercent, color);
const branches = coverageThreshold(coveredBranchPercent, color);
const functions = coverageThreshold(coveredFunctionPercent, color);
const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');

report += `${pad}${symbol}${relativePath} | ${lines} | ${branches} | ` +
`${functions} | ${uncovered}\n`;
}

const { totals } = summary;
report += `${pad}${symbol}all files | ` +
`${coverageThreshold(totals.coveredLinePercent, color)} | ` +
`${coverageThreshold(totals.coveredBranchPercent, color)} | ` +
`${coverageThreshold(totals.coveredFunctionPercent, color)} |\n`;

report += `${pad}${symbol}end of coverage report\n`;
if (color) {
report += white;
}
return report;
}

module.exports = {
convertStringToRegExp,
countCompletedTest,
Expand All @@ -255,4 +304,5 @@ module.exports = {
isTestFailureError,
parseCommandLine,
setupTestReporters,
getCoverageReport,
};
156 changes: 106 additions & 50 deletions test/parallel/test-runner-coverage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ function findCoverageFileForPid(pid) {
});
}

function getCoverageFixtureReport() {
function getTapCoverageFixtureReport() {
const report = [
'# start of coverage report',
'# file | line % | branch % | funcs % | uncovered lines',
Expand All @@ -37,64 +37,120 @@ function getCoverageFixtureReport() {
return report;
}

test('--experimental-test-coverage and --test cannot be combined', () => {
// TODO(cjihrig): This test can be removed once multi-process code coverage
// is supported.
const args = ['--test', '--experimental-test-coverage'];
const result = spawnSync(process.execPath, args);

// 9 is the documented exit code for an invalid CLI argument.
assert.strictEqual(result.status, 9);
assert.match(
result.stderr.toString(),
/--experimental-test-coverage cannot be used with --test/
);
});
function getSpecCoverageFixtureReport() {
const report = [
'\u2139 start of coverage report',
'\u2139 file | line % | branch % | funcs % | uncovered lines',
'\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12, ' +
'13, 16, 17, 18, 19, 20, 21, 22, 27, 39, 43, 44, 61, 62, 66, 67, 71, 72',
'\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ',
'\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5, 6',
'\u2139 all files | 78.35 | 43.75 | 60.00 |',
'\u2139 end of coverage report',
].join('\n');

test('handles the inspector not being available', (t) => {
if (process.features.inspector) {
return;
if (common.isWindows) {
return report.replaceAll('/', '\\');
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);
return report;
}

assert(!result.stdout.toString().includes('# start of coverage report'));
assert(result.stderr.toString().includes('coverage could not be collected'));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
test('test coverage report', async (t) => {
await t.test('--experimental-test-coverage and --test cannot be combined', () => {
// TODO(cjihrig): This test can be removed once multi-process code coverage
// is supported.
const args = ['--test', '--experimental-test-coverage'];
const result = spawnSync(process.execPath, args);

// 9 is the documented exit code for an invalid CLI argument.
assert.strictEqual(result.status, 9);
assert.match(
result.stderr.toString(),
/--experimental-test-coverage cannot be used with --test/
);
});

test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}
await t.test('handles the inspector not being available', (t) => {
if (process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getCoverageFixtureReport();
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
assert(!result.stdout.toString().includes('# start of coverage report'));
assert(result.stderr.toString().includes('coverage could not be collected'));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});

test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}
test('test tap coverage reporter', async (t) => {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not something that needs to be done in this PR, but we should probably specify --test-reporter tap explicitly for these tests now that the default is sometimes TAP and sometimes spec.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can do in this pr 👍🏼

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not something that needs to be done in this PR, but we should probably specify --test-reporter tap explicitly for these tests now that the default is sometimes TAP and sometimes spec.

Since tap is more machine friendly and spec more user friendly, I think the latest one should be used by default always, and being tap and opt-in, similar to json output.

await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getTapCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
});

await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
const result = spawnSync(process.execPath, args);
const report = getTapCoverageFixtureReport();

const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', fixture];
const result = spawnSync(process.execPath, args);
const report = getCoverageFixtureReport();
assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
test('test spec coverage reporter', async (t) => {
await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
if (!process.features.inspector) {
return;
}
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
const result = spawnSync(process.execPath, args, options);
const report = getSpecCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(findCoverageFileForPid(result.pid));
});

await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
if (!process.features.inspector) {
return;
}
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
const result = spawnSync(process.execPath, args);
const report = getSpecCoverageFixtureReport();

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.stderr.toString(), '');
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});
});