diff --git a/Makefile b/Makefile index cc6fcd888f4cde..e003341f70cf51 100644 --- a/Makefile +++ b/Makefile @@ -496,6 +496,9 @@ test-debug: test-build test-message: test-build $(PYTHON) tools/test.py $(PARALLEL_ARGS) message +test-wpt: all + $(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt + test-simple: | cctest bench-addons-build # Depends on 'all'. $(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential diff --git a/test/common/README.md b/test/common/README.md index f0bcced82e6149..356c8531ec0c38 100644 --- a/test/common/README.md +++ b/test/common/README.md @@ -772,12 +772,19 @@ Deletes and recreates the testing temporary directory. ## WPT Module -The wpt.js module is a port of parts of -[W3C testharness.js](https://github.com/w3c/testharness.js) for testing the -Node.js -[WHATWG URL API](https://nodejs.org/api/url.html#url_the_whatwg_url_api) -implementation with tests from -[W3C Web Platform Tests](https://github.com/w3c/web-platform-tests). +### harness + +A legacy port of [Web Platform Tests][] harness. + +See the source code for definitions. Please avoid using it in new +code - the current usage of this port in tests is being migrated to +the original WPT harness, see [the WPT tests README][]. + +### Class: WPTRunner + +A driver class for running WPT with the WPT harness in a vm. + +See [the WPT tests README][] for details. [<Array>]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array @@ -792,3 +799,5 @@ implementation with tests from [`hijackstdio.hijackStdErr()`]: #hijackstderrlistener [`hijackstdio.hijackStdOut()`]: #hijackstdoutlistener [internationalization]: https://github.com/nodejs/node/wiki/Intl +[Web Platform Tests]: https://github.com/web-platform-tests/wpt +[the WPT tests README]: ../wpt/README.md diff --git a/test/common/wpt.js b/test/common/wpt.js index 7cd644dc88c097..59dbe26d2abdcb 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -2,9 +2,17 @@ 'use strict'; const assert = require('assert'); +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const fs = require('fs'); +const fsPromises = fs.promises; +const path = require('path'); +const vm = require('vm'); // https://github.com/w3c/testharness.js/blob/master/testharness.js -module.exports = { +// TODO: get rid of this half-baked harness in favor of the one +// pulled from WPT +const harnessMock = { test: (fn, desc) => { try { fn(); @@ -28,3 +36,420 @@ module.exports = { assert.fail(`Reached unreachable code: ${desc}`); } }; + +class ResourceLoader { + constructor(path) { + this.path = path; + } + + fetch(url, asPromise = true) { + // We need to patch this to load the WebIDL parser + url = url.replace( + '/resources/WebIDLParser.js', + '/resources/webidl2/lib/webidl2.js' + ); + const file = url.startsWith('/') ? + fixtures.path('wpt', url) : + fixtures.path('wpt', this.path, url); + if (asPromise) { + return fsPromises.readFile(file) + .then((data) => { + return { + ok: true, + json() { return JSON.parse(data.toString()); }, + text() { return data.toString(); } + }; + }); + } else { + return fs.readFileSync(file, 'utf8'); + } + } +} + +class WPTTest { + /** + * @param {string} mod + * @param {string} filename + * @param {string[]} requires + * @param {string | undefined} failReason + * @param {string | undefined} skipReason + */ + constructor(mod, filename, requires, failReason, skipReason) { + this.module = mod; // name of the WPT module, e.g. 'url' + this.filename = filename; // name of the test file + this.requires = requires; + this.failReason = failReason; + this.skipReason = skipReason; + } + + getAbsolutePath() { + return fixtures.path('wpt', this.module, this.filename); + } + + getContent() { + return fs.readFileSync(this.getAbsolutePath(), 'utf8'); + } + + shouldSkip() { + return this.failReason || this.skipReason; + } + + requireIntl() { + return this.requires.includes('intl'); + } +} + +class StatusLoader { + constructor(path) { + this.path = path; + this.loaded = false; + this.status = null; + /** @type {WPTTest[]} */ + this.tests = []; + } + + loadTest(file) { + let requires = []; + let failReason; + let skipReason; + if (this.status[file]) { + requires = this.status[file].requires || []; + failReason = this.status[file].fail; + skipReason = this.status[file].skip; + } + return new WPTTest(this.path, file, requires, + failReason, skipReason); + } + + load() { + const dir = path.join(__dirname, '..', 'wpt'); + const statusFile = path.join(dir, 'status', `${this.path}.json`); + const result = JSON.parse(fs.readFileSync(statusFile, 'utf8')); + this.status = result; + + const list = fs.readdirSync(fixtures.path('wpt', this.path)); + + for (const file of list) { + this.tests.push(this.loadTest(file)); + } + this.loaded = true; + } + + get jsTests() { + return this.tests.filter((test) => test.filename.endsWith('.js')); + } +} + +const PASSED = 1; +const FAILED = 2; +const SKIPPED = 3; + +class WPTRunner { + constructor(path) { + this.path = path; + this.resource = new ResourceLoader(path); + this.sandbox = null; + this.context = null; + + this.globals = new Map(); + + this.status = new StatusLoader(path); + this.status.load(); + this.tests = new Map( + this.status.jsTests.map((item) => [item.filename, item]) + ); + + this.results = new Map(); + this.inProgress = new Set(); + } + + /** + * Specify that certain global descriptors from the object + * should be defined in the vm + * @param {object} obj + * @param {string[]} names + */ + copyGlobalsFromObject(obj, names) { + for (const name of names) { + const desc = Object.getOwnPropertyDescriptor(global, name); + this.globals.set(name, desc); + } + } + + /** + * Specify that certain global descriptors should be defined in the vm + * @param {string} name + * @param {object} descriptor + */ + defineGlobal(name, descriptor) { + this.globals.set(name, descriptor); + } + + // TODO(joyeecheung): work with the upstream to port more tests in .html + // to .js. + runJsTests() { + // TODO(joyeecheung): it's still under discussion whether we should leave + // err.name alone. See https://github.com/nodejs/node/issues/20253 + const internalErrors = require('internal/errors'); + internalErrors.useOriginalName = true; + + let queue = []; + + // If the tests are run as `node test/wpt/test-something.js subset.any.js`, + // only `subset.any.js` will be run by the runner. + if (process.argv[2]) { + const filename = process.argv[2]; + if (!this.tests.has(filename)) { + throw new Error(`${filename} not found!`); + } + queue.push(this.tests.get(filename)); + } else { + queue = this.buildQueue(); + } + + this.inProgress = new Set(queue.map((item) => item.filename)); + + for (const test of queue) { + const filename = test.filename; + const content = test.getContent(); + const meta = test.title = this.getMeta(content); + + const absolutePath = test.getAbsolutePath(); + const context = this.generateContext(test.filename); + const code = this.mergeScripts(meta, content); + try { + vm.runInContext(code, context, { + filename: absolutePath + }); + } catch (err) { + this.fail(filename, { + name: '', + message: err.message, + stack: err.stack + }, 'UNCAUGHT'); + this.inProgress.delete(filename); + } + } + this.tryFinish(); + } + + mock() { + const resource = this.resource; + const result = { + // This is a mock, because at the moment fetch is not implemented + // in Node.js, but some tests and harness depend on this to pull + // resources. + fetch(file) { + return resource.fetch(file); + }, + location: {}, + GLOBAL: { + isWindow() { return false; } + }, + Object + }; + + return result; + } + + // Note: this is how our global space for the WPT test should look like + getSandbox() { + const result = this.mock(); + for (const [name, desc] of this.globals) { + Object.defineProperty(result, name, desc); + } + return result; + } + + generateContext(filename) { + const sandbox = this.sandbox = this.getSandbox(); + const context = this.context = vm.createContext(sandbox); + + const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); + const harness = fs.readFileSync(harnessPath, 'utf8'); + vm.runInContext(harness, context, { + filename: harnessPath + }); + + sandbox.add_result_callback( + this.resultCallback.bind(this, filename) + ); + sandbox.add_completion_callback( + this.completionCallback.bind(this, filename) + ); + sandbox.self = sandbox; + // TODO(joyeecheung): we are not a window - work with the upstream to + // add a new scope for us. + sandbox.document = {}; // Pretend we are Window + return context; + } + + resultCallback(filename, test) { + switch (test.status) { + case 1: + this.fail(filename, test, 'FAILURE'); + break; + case 2: + this.fail(filename, test, 'TIMEOUT'); + break; + case 3: + this.fail(filename, test, 'INCOMPLETE'); + break; + default: + this.succeed(filename, test); + } + } + + completionCallback(filename, tests, harnessStatus) { + if (harnessStatus.status === 2) { + assert.fail(`test harness timed out in ${filename}`); + } + this.inProgress.delete(filename); + this.tryFinish(); + } + + tryFinish() { + if (this.inProgress.size > 0) { + return; + } + + this.reportResults(); + } + + reportResults() { + const unexpectedFailures = []; + for (const [filename, items] of this.results) { + const test = this.tests.get(filename); + let title = test.meta && test.meta.title; + title = title ? `${filename} : ${title}` : filename; + console.log(`---- ${title} ----`); + for (const item of items) { + switch (item.type) { + case FAILED: { + if (test.failReason) { + console.log(`[EXPECTED_FAILURE] ${item.test.name}`); + } else { + console.log(`[UNEXPECTED_FAILURE] ${item.test.name}`); + unexpectedFailures.push([title, filename, item]); + } + break; + } + case PASSED: { + console.log(`[PASSED] ${item.test.name}`); + break; + } + case SKIPPED: { + console.log(`[SKIPPED] ${item.reason}`); + break; + } + } + } + } + + if (unexpectedFailures.length > 0) { + for (const [title, filename, item] of unexpectedFailures) { + console.log(`---- ${title} ----`); + console.log(`[${item.reason}] ${item.test.name}`); + console.log(item.test.message); + console.log(item.test.stack); + const command = `${process.execPath} ${process.execArgv}` + + ` ${require.main.filename} ${filename}`; + console.log(`Command: ${command}\n`); + } + assert.fail(`${unexpectedFailures.length} unexpected failures found`); + } + } + + addResult(filename, item) { + const result = this.results.get(filename); + if (result) { + result.push(item); + } else { + this.results.set(filename, [item]); + } + } + + succeed(filename, test) { + this.addResult(filename, { + type: PASSED, + test + }); + } + + fail(filename, test, reason) { + this.addResult(filename, { + type: FAILED, + test, + reason + }); + } + + skip(filename, reason) { + this.addResult(filename, { + type: SKIPPED, + reason + }); + } + + getMeta(code) { + const matches = code.match(/\/\/ META: .+/g); + if (!matches) { + return {}; + } else { + const result = {}; + for (const match of matches) { + const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/); + const key = parts[1]; + const value = parts[2]; + if (key === 'script') { + if (result[key]) { + result[key].push(value); + } else { + result[key] = [value]; + } + } else { + result[key] = value; + } + } + return result; + } + } + + mergeScripts(meta, content) { + if (!meta.script) { + return content; + } + + // only one script + let result = ''; + for (const script of meta.script) { + result += this.resource.fetch(script, false); + } + + return result + content; + } + + buildQueue() { + const queue = []; + for (const test of this.tests.values()) { + const filename = test.filename; + if (test.skipReason) { + this.skip(filename, test.skipReason); + continue; + } + + if (!common.hasIntl && test.requireIntl()) { + this.skip(filename, 'missing Intl'); + continue; + } + + queue.push(test); + } + return queue; + } +} + +module.exports = { + harness: harnessMock, + WPTRunner +}; diff --git a/test/parallel/test-whatwg-url-constructor.js b/test/parallel/test-whatwg-url-constructor.js index 6570e36b0fd94e..282679d7797f2a 100644 --- a/test/parallel/test-whatwg-url-constructor.js +++ b/test/parallel/test-whatwg-url-constructor.js @@ -8,7 +8,7 @@ if (!common.hasIntl) { const fixtures = require('../common/fixtures'); const { URL, URLSearchParams } = require('url'); const { test, assert_equals, assert_true, assert_throws } = - require('../common/wpt'); + require('../common/wpt').harness; const request = { response: require( diff --git a/test/parallel/test-whatwg-url-custom-searchparams-sort.js b/test/parallel/test-whatwg-url-custom-searchparams-sort.js index f8884a7e7092a3..49c3b065f957c6 100644 --- a/test/parallel/test-whatwg-url-custom-searchparams-sort.js +++ b/test/parallel/test-whatwg-url-custom-searchparams-sort.js @@ -4,9 +4,14 @@ require('../common'); const { URL, URLSearchParams } = require('url'); -const { test, assert_array_equals } = require('../common/wpt'); +const { test, assert_array_equals } = require('../common/wpt').harness; -// Test bottom-up iterative stable merge sort +// TODO(joyeecheung): upstream this to WPT, if possible - even +// just as a test for large inputs. Other implementations may +// have a similar cutoff anyway. + +// Test bottom-up iterative stable merge sort because we only use that +// algorithm to sort > 100 search params. const tests = [{ input: '', output: [] }]; const pairs = []; for (let i = 10; i < 100; i++) { diff --git a/test/parallel/test-whatwg-url-custom-setters.js b/test/parallel/test-whatwg-url-custom-setters.js index 99b4361831fd3b..e10ebb9fe66968 100644 --- a/test/parallel/test-whatwg-url-custom-setters.js +++ b/test/parallel/test-whatwg-url-custom-setters.js @@ -10,9 +10,10 @@ if (!common.hasIntl) { const assert = require('assert'); const URL = require('url').URL; -const { test, assert_equals } = require('../common/wpt'); +const { test, assert_equals } = require('../common/wpt').harness; const fixtures = require('../common/fixtures'); +// TODO(joyeecheung): we should submit these to the upstream const additionalTestCases = require(fixtures.path('url-setter-tests-additional.js')); diff --git a/test/parallel/test-whatwg-url-origin.js b/test/parallel/test-whatwg-url-origin.js index 0ce19c28218779..5b1aa14cd07642 100644 --- a/test/parallel/test-whatwg-url-origin.js +++ b/test/parallel/test-whatwg-url-origin.js @@ -7,7 +7,7 @@ if (!common.hasIntl) { const fixtures = require('../common/fixtures'); const URL = require('url').URL; -const { test, assert_equals } = require('../common/wpt'); +const { test, assert_equals } = require('../common/wpt').harness; const request = { response: require( diff --git a/test/parallel/test-whatwg-url-setters.js b/test/parallel/test-whatwg-url-setters.js index a5d59f761c1a13..13422bd77690ed 100644 --- a/test/parallel/test-whatwg-url-setters.js +++ b/test/parallel/test-whatwg-url-setters.js @@ -7,7 +7,7 @@ if (!common.hasIntl) { } const URL = require('url').URL; -const { test, assert_equals } = require('../common/wpt'); +const { test, assert_equals } = require('../common/wpt').harness; const fixtures = require('../common/fixtures'); const request = { diff --git a/test/parallel/test-whatwg-url-toascii.js b/test/parallel/test-whatwg-url-toascii.js index 74ce1fe6c2f281..4869ab3f1d3bd1 100644 --- a/test/parallel/test-whatwg-url-toascii.js +++ b/test/parallel/test-whatwg-url-toascii.js @@ -7,7 +7,7 @@ if (!common.hasIntl) { const fixtures = require('../common/fixtures'); const { URL } = require('url'); -const { test, assert_equals, assert_throws } = require('../common/wpt'); +const { test, assert_equals, assert_throws } = require('../common/wpt').harness; const request = { response: require( diff --git a/test/wpt/README.md b/test/wpt/README.md new file mode 100644 index 00000000000000..1810a98c8dc982 --- /dev/null +++ b/test/wpt/README.md @@ -0,0 +1,171 @@ +# Web Platform Tests + +The tests here are drivers for running the [Web Platform Tests][]. + +See [`test/fixtures/wpt/README.md`][] for a hash of the last +updated WPT commit for each module being covered here. + +See the json files in [the `status` folder](./status) for prerequisites, +expected failures, and support status for specific tests in each module. + +Currently there are still some Web Platform Tests titled `test-whatwg-*` +under `test/parallel` that have not been migrated to be run with the +WPT harness and have automatic updates. There are also a few +`test-whatwg-*-custom-*` tests that may need to be upstreamed. +This folder covers the tests that have been migrated. + + +## How to add tests for a new module + +### 1. Create a status file + +For example, to add the URL tests, add a `test/wpt/status/url.json` file. + +In the beginning, it's fine to leave an empty object `{}` in the file if +it's not yet clear how compliant the implementation is, +the requirements and expected failures can be figured out in a later step +when the tests are run for the first time. + +See [Format of a status JSON file](#status-format) for details. + +### 2. Pull the WPT files + +Use the [git node wpt][] command to download the WPT files into +`test/fixtures/wpt`. For example, to add URL tests: + +```text +$ cd /path/to/node/project +$ git node wpt url +``` + +### 3. Create the test driver + +For example, for the URL tests, add a file `test/wpt/test-whatwg-url.js`: + +```js +'use strict'; + +// This flag is required by the WPT Runner to patch the internals +// for the tests to run in a vm. +// Flags: --expose-internals + +require('../common'); +const { WPTRunner } = require('../common/wpt'); + +const runner = new WPTRunner('url'); + +// Copy global descriptors from the global object +runner.copyGlobalsFromObject(global, ['URL', 'URLSearchParams']); +// Define any additional globals with descriptors +runner.defineGlobal('DOMException', { + get() { + return require('internal/domexception'); + } +}); + +runner.runJsTests(); +``` + +This driver is capable of running the tests located in `test/fixtures/wpt/url` +with the WPT harness while taking the status file into account. + +### 4. Run the tests + +Run the test using `tools/test.py` and see if there are any failures. +For example, to run all the URL tests under `test/fixtures/wpt/url`: + +```text +$ tools/test.py wpt/test-whatwg-url +``` + +To run a specific test in WPT, for example, `url/url-searchparams.any.js`, +pass the file name as argument to the corresponding test driver: + +```text +node --expose-internals test/wpt/test-whatwg-url.js url-searchparams.any.js +``` + +If there are any failures, update the corresponding status file +(in this case, `test/wpt/status/url.json`) to make the test pass. + +For example, to mark `url/url-searchparams.any.js` as expected to fail, +add this to `test/wpt/status/url.json`: + +```json + "url-searchparams.any.js": { + "fail": "explain why the test fails, ideally with links" + } +``` + +See [Format of a status JSON file](#status-format) for details. + +### 5. Commit the changes and submit a Pull Request + +See [the contributing guide](../../CONTRIBUTING.md). + +## How to update tests for a module + +The tests can be updated in a way similar to how they are added. +Run Step 2 and Step 4 of [adding tests for a new module](#add-tests). + +The [git node wpt][] command maintains the status of the local +WPT subset, if no files are updated after running it for a module, +the local subset is up to date and there is no need to update them +until they are changed in the upstream. + +## How it works + +Note: currently this test suite only supports `.js` tests. There is +ongoing work in the upstream to properly split out the tests into files +that can be run in a shell environment like Node.js. + +### Getting the original test files and harness from WPT + +The original files and harness from WPT are downloaded and stored in +`test/fixtures/wpt`. + +The [git node wpt][] command automate this process while maintaining a map +containing the hash of the last updated commit for each module in +`test/fixtures/wpt/versions.json` and [`test/fixtures/wpt/README.md`][]. +It also maintains the LICENSE file in `test/fixtures/wpt`. + +### Loading and running the tests + +Given a module, the `WPTRunner` class in [`test/common/wpt`](../common/wpt.js) +loads: + +- `.js` test files (for example, `test/common/wpt/url/*.js` for `url`) +- Status file (for example, `test/wpt/status/url.json` for `url`) +- The WPT harness + +Then, for each test, it creates a vm with the globals and mocks, +sets up the harness result hooks, loads the metadata in the test (including +loading extra resources), and runs all the tests in that vm, +skipping tests that cannot be run because of lack of dependency or +expected failures. + + +## Format of a status JSON file + +```text +{ + "something.scope.js": { // the file name + // Optional: If the requirement is not met, this test will be skipped + "requires": ["intl"], // currently only intl is supported + + // Optional: the test will be skipped with the reason printed + "skip": "explain why we cannot run a test that's supposed to pass", + + // Optional: the test will be skipped with the reason printed + "fail": "explain why we the test is expected to fail" + } +} +``` + +A test may have to be skipped because it depends on another irrelevant +Web API, or certain harness has not been ported in our test runner yet. +In that case it needs to be marked with `skip` instead of `fail`. + +[Web Platform Tests]: https://github.com/web-platform-tests/wpt +[git node wpt]: https://github.com/nodejs/node-core-utils/blob/master/docs/git-node.md#git-node-wpt +[`test/fixtures/wpt/README.md`]: ../fixtures/wpt/README.md diff --git a/test/wpt/status/console.json b/test/wpt/status/console.json new file mode 100644 index 00000000000000..3874ed148d8610 --- /dev/null +++ b/test/wpt/status/console.json @@ -0,0 +1,8 @@ +{ + "idlharness.any.js": { + "fail": ".table, .dir and .timeLog parameter lengths are wrong" + }, + "console-is-a-namespace.any.js": { + "fail": "The [[Prototype]]'s [[Prototype]] must be %ObjectPrototype%" + } +} diff --git a/test/wpt/status/url.json b/test/wpt/status/url.json new file mode 100644 index 00000000000000..42b5f6fb1a1146 --- /dev/null +++ b/test/wpt/status/url.json @@ -0,0 +1,15 @@ +{ + "toascii.window.js": { + "requires": ["intl"], + "skip": "TODO: port from .window.js" + }, + "historical.any.js": { + "requires": ["intl"] + }, + "urlencoded-parser.any.js": { + "fail": "missing Request and Response" + }, + "idlharness.any.js": { + "fail": "getter/setter names are wrong, etc." + } +} \ No newline at end of file diff --git a/test/wpt/test-whatwg-console.js b/test/wpt/test-whatwg-console.js new file mode 100644 index 00000000000000..7b23fe8d3e619d --- /dev/null +++ b/test/wpt/test-whatwg-console.js @@ -0,0 +1,13 @@ +'use strict'; + +// Flags: --expose-internals + +require('../common'); +const { WPTRunner } = require('../common/wpt'); + +const runner = new WPTRunner('console'); + +// Copy global descriptors from the global object +runner.copyGlobalsFromObject(global, ['console']); + +runner.runJsTests(); diff --git a/test/wpt/test-whatwg-url.js b/test/wpt/test-whatwg-url.js new file mode 100644 index 00000000000000..8734452940e84e --- /dev/null +++ b/test/wpt/test-whatwg-url.js @@ -0,0 +1,19 @@ +'use strict'; + +// Flags: --expose-internals + +require('../common'); +const { WPTRunner } = require('../common/wpt'); + +const runner = new WPTRunner('url'); + +// Copy global descriptors from the global object +runner.copyGlobalsFromObject(global, ['URL', 'URLSearchParams']); +// Needed by urlsearchparams-constructor.any.js +runner.defineGlobal('DOMException', { + get() { + return require('internal/domexception'); + } +}); + +runner.runJsTests(); diff --git a/test/wpt/testcfg.py b/test/wpt/testcfg.py new file mode 100644 index 00000000000000..db235699ddfe57 --- /dev/null +++ b/test/wpt/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.ParallelTestConfiguration(context, root, 'wpt') diff --git a/test/wpt/wpt.status b/test/wpt/wpt.status new file mode 100644 index 00000000000000..de3b3024627a1b --- /dev/null +++ b/test/wpt/wpt.status @@ -0,0 +1,21 @@ +prefix wpt + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$arch==arm || $arch==arm64] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + +[$system==aix]