From 80701d8861c12c086e1395af10e7c82ddd8e94e6 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Thu, 13 Apr 2023 21:39:58 -0400 Subject: [PATCH] Rename test files. - Rename test.js to test-node.js. Aligns with test-karma.js. - Rename test-common.js to test.js. --- package.json | 2 +- tests/test-common.js | 1113 ------------------------------------- tests/test-karma.js | 2 +- tests/test-node.js | 213 +++++++ tests/test.js | 1266 ++++++++++++++++++++++++++++++++++++------ 5 files changed, 1298 insertions(+), 1298 deletions(-) delete mode 100644 tests/test-common.js create mode 100644 tests/test-node.js diff --git a/package.json b/package.json index 1a7b531e..7a0bd8ee 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "fetch-json-ld-org-test-suite": "if [ ! -e test-suites/json-ld.org ]; then git clone --depth 1 https://github.com/json-ld/json-ld.org.git test-suites/json-ld.org; fi", "fetch-rdf-canon-test-suite": "if [ ! -e test-suites/rdf-canon ]; then git clone --depth 1 https://github.com/w3c/rdf-canon.git test-suites/rdf-canon; fi", "test": "npm run test-node", - "test-node": "cross-env NODE_ENV=test mocha --delay -t 30000 -A -R ${REPORTER:-spec} tests/test.js", + "test-node": "cross-env NODE_ENV=test mocha --delay -t 30000 -A -R ${REPORTER:-spec} tests/test-node.js", "test-karma": "cross-env NODE_ENV=test karma start", "coverage": "cross-env NODE_ENV=test nyc --reporter=lcov --reporter=text-summary npm test", "coverage-ci": "cross-env NODE_ENV=test nyc --reporter=lcovonly npm run test", diff --git a/tests/test-common.js b/tests/test-common.js deleted file mode 100644 index 500fe9ed..00000000 --- a/tests/test-common.js +++ /dev/null @@ -1,1113 +0,0 @@ -/** - * Copyright (c) 2011-2019 Digital Bazaar, Inc. All rights reserved. - */ -/* eslint-disable indent */ -const EarlReport = require('./earl-report'); -const join = require('join-path-js'); -const rdfCanonize = require('rdf-canonize'); -const {prependBase} = require('../lib/url'); -const {klona} = require('klona'); - -module.exports = function(options) { - -'use strict'; - -const assert = options.assert; -const benchmark = options.benchmark; -const jsonld = options.jsonld; - -const manifest = options.manifest || { - '@context': 'https://json-ld.org/test-suite/context.jsonld', - '@id': '', - '@type': 'mf:Manifest', - description: 'Top level jsonld.js manifest', - name: 'jsonld.js', - sequence: options.entries || [], - filename: '/' -}; - -const TEST_TYPES = { - 'jld:CompactTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - /compact-manifest#t0112$/, - /compact-manifest#t0113$/, - // html - /html-manifest#tc001$/, - /html-manifest#tc002$/, - /html-manifest#tc003$/, - /html-manifest#tc004$/, - ] - }, - fn: 'compact', - params: [ - readTestUrl('input'), - readTestJson('context'), - createTestOptions() - ], - compare: compareExpectedJson - }, - 'jld:ExpandTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - // spec issues - // Unclear how to handle {"@id": null} edge case - // See https://github.com/w3c/json-ld-api/issues/480 - // non-normative test, also see toRdf-manifest#te122 - ///expand-manifest#t0122$/, - - // misc - /expand-manifest#tc037$/, - /expand-manifest#tc038$/, - /expand-manifest#ter54$/, - - // html - /html-manifest#te001$/, - /html-manifest#te002$/, - /html-manifest#te003$/, - /html-manifest#te004$/, - /html-manifest#te005$/, - /html-manifest#te006$/, - /html-manifest#te007$/, - /html-manifest#te010$/, - /html-manifest#te011$/, - /html-manifest#te012$/, - /html-manifest#te013$/, - /html-manifest#te014$/, - /html-manifest#te015$/, - /html-manifest#te016$/, - /html-manifest#te017$/, - /html-manifest#te018$/, - /html-manifest#te019$/, - /html-manifest#te020$/, - /html-manifest#te021$/, - /html-manifest#te022$/, - /html-manifest#tex01$/, - // HTML extraction - /expand-manifest#thc01$/, - /expand-manifest#thc02$/, - /expand-manifest#thc03$/, - /expand-manifest#thc04$/, - /expand-manifest#thc05$/, - // remote - /remote-doc-manifest#t0013$/, // HTML - ] - }, - fn: 'expand', - params: [ - readTestUrl('input'), - createTestOptions() - ], - compare: compareExpectedJson - }, - 'jld:FlattenTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - // html - /html-manifest#tf001$/, - /html-manifest#tf002$/, - /html-manifest#tf003$/, - /html-manifest#tf004$/, - ] - }, - fn: 'flatten', - params: [ - readTestUrl('input'), - readTestJson('context'), - createTestOptions() - ], - compare: compareExpectedJson - }, - 'jld:FrameTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - /frame-manifest#t0069$/, - ] - }, - fn: 'frame', - params: [ - readTestUrl('input'), - readTestJson('frame'), - createTestOptions() - ], - compare: compareExpectedJson - }, - 'jld:FromRDFTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - // direction (compound-literal) - /fromRdf-manifest#tdi11$/, - /fromRdf-manifest#tdi12$/, - ] - }, - fn: 'fromRDF', - params: [ - readTestNQuads('input'), - createTestOptions({format: 'application/n-quads'}) - ], - compare: compareExpectedJson - }, - 'jld:NormalizeTest': { - fn: 'normalize', - params: [ - readTestUrl('input'), - createTestOptions({format: 'application/n-quads'}) - ], - compare: compareExpectedNQuads - }, - 'jld:ToRDFTest': { - skip: { - // skip tests where behavior changed for a 1.1 processor - // see JSON-LD 1.0 Errata - specVersion: ['json-ld-1.0'], - // FIXME - // NOTE: idRegex format: - //MMM-manifest#tNNN$/, - idRegex: [ - // spec issues - // Unclear how to handle {"@id": null} edge case - // See https://github.com/w3c/json-ld-api/issues/480 - // normative test, also see expand-manifest#t0122 - ///toRdf-manifest#te122$/, - - // misc - /toRdf-manifest#tc037$/, - /toRdf-manifest#tc038$/, - /toRdf-manifest#ter54$/, - /toRdf-manifest#tli12$/, - /toRdf-manifest#tli14$/, - - // well formed - /toRdf-manifest#twf05$/, - - // html - /html-manifest#tr001$/, - /html-manifest#tr002$/, - /html-manifest#tr003$/, - /html-manifest#tr004$/, - /html-manifest#tr005$/, - /html-manifest#tr006$/, - /html-manifest#tr007$/, - /html-manifest#tr010$/, - /html-manifest#tr011$/, - /html-manifest#tr012$/, - /html-manifest#tr013$/, - /html-manifest#tr014$/, - /html-manifest#tr015$/, - /html-manifest#tr016$/, - /html-manifest#tr017$/, - /html-manifest#tr018$/, - /html-manifest#tr019$/, - /html-manifest#tr020$/, - /html-manifest#tr021$/, - /html-manifest#tr022$/, - // Invalid Statement - /toRdf-manifest#te075$/, - /toRdf-manifest#te111$/, - /toRdf-manifest#te112$/, - // direction (compound-literal) - /toRdf-manifest#tdi11$/, - /toRdf-manifest#tdi12$/, - ] - }, - fn: 'toRDF', - params: [ - readTestUrl('input'), - createTestOptions({format: 'application/n-quads'}) - ], - compare: compareCanonizedExpectedNQuads - }, - 'rdfc:Urgna2012EvalTest': { - fn: 'normalize', - params: [ - readTestNQuads('action'), - createTestOptions({ - algorithm: 'URGNA2012', - inputFormat: 'application/n-quads', - format: 'application/n-quads' - }) - ], - compare: compareExpectedNQuads - }, - 'rdfc:Urdna2015EvalTest': { - skip: { - // NOTE: idRegex format: - //manifest-urdna2015#testNNN$/, - idRegex: [ - // Unsupported U escape - /manifest-urdna2015#test060/ - ] - }, - fn: 'normalize', - params: [ - readTestNQuads('action'), - createTestOptions({ - algorithm: 'URDNA2015', - inputFormat: 'application/n-quads', - format: 'application/n-quads' - }) - ], - compare: compareExpectedNQuads - } -}; - -const SKIP_TESTS = []; - -// create earl report -if(options.earl && options.earl.filename) { - options.earl.report = new EarlReport({ - env: options.testEnv - }); - if(options.benchmarkOptions) { - options.earl.report.setupForBenchmarks({testEnv: options.testEnv}); - } -} - -return new Promise(resolve => { - -// async generated tests -// _tests => [{suite}, ...] -// suite => { -// title: ..., -// tests: [test, ...], -// suites: [suite, ...] -// } -const _tests = []; - -return addManifest(manifest, _tests) - .then(() => { - return _testsToMocha(_tests); - }).then(result => { - if(options.earl.report) { - describe('Writing EARL report to: ' + options.earl.filename, function() { - // print out EARL even if .only was used - const _it = result.hadOnly ? it.only : it; - _it('should print the earl report', function() { - return options.writeFile( - options.earl.filename, options.earl.report.reportJson()); - }); - }); - } - }).then(() => resolve()); - -// build mocha tests from local test structure -function _testsToMocha(tests) { - let hadOnly = false; - tests.forEach(suite => { - if(suite.skip) { - describe.skip(suite.title); - return; - } - describe(suite.title, () => { - suite.tests.forEach(test => { - if(test.only) { - hadOnly = true; - it.only(test.title, test.f); - return; - } - it(test.title, test.f); - }); - const {hadOnly: _hadOnly} = _testsToMocha(suite.suites); - hadOnly = hadOnly || _hadOnly; - }); - suite.imports.forEach(f => { - options.import(f); - }); - }); - return { - hadOnly - }; -} - -}); - -/** - * Adds the tests for all entries in the given manifest. - * - * @param manifest {Object} the manifest. - * @param parent {Object} the parent test structure - * @return {Promise} - */ -function addManifest(manifest, parent) { - return new Promise((resolve, reject) => { - // create test structure - const suite = { - title: manifest.name || manifest.label, - tests: [], - suites: [], - imports: [] - }; - parent.push(suite); - - // get entries and sequence (alias for entries) - const entries = [].concat( - getJsonLdValues(manifest, 'entries'), - getJsonLdValues(manifest, 'sequence') - ); - - const includes = getJsonLdValues(manifest, 'include'); - // add includes to sequence as jsonld files - for(let i = 0; i < includes.length; ++i) { - entries.push(includes[i] + '.jsonld'); - } - - // resolve all entry promises and process - Promise.all(entries).then(entries => { - let p = Promise.resolve(); - entries.forEach(entry => { - if(typeof entry === 'string' && entry.endsWith('js')) { - // process later as a plain JavaScript file - suite.imports.push(entry); - return; - } else if(typeof entry === 'function') { - // process as a function that returns a promise - p = p.then(() => { - return entry(options); - }).then(childSuite => { - if(suite) { - suite.suites.push(childSuite); - } - }); - return; - } - p = p.then(() => { - return readManifestEntry(manifest, entry); - }).then(entry => { - if(isJsonLdType(entry, '__SKIP__')) { - // special local skip logic - suite.tests.push(entry); - } else if(isJsonLdType(entry, 'mf:Manifest')) { - // entry is another manifest - return addManifest(entry, suite.suites); - } else { - // assume entry is a test - return addTest(manifest, entry, suite.tests); - } - }); - }); - return p; - }).then(() => { - resolve(); - }).catch(err => { - console.error(err); - reject(err); - }); - }); -} - -/** - * Adds a test. - * - * @param manifest {Object} the manifest. - * @param test {Object} the test. - * @param tests {Array} the list of tests to add to. - * @return {Promise} - */ -async function addTest(manifest, test, tests) { - // expand @id and input base - const test_id = test['@id'] || test.id; - //var number = test_id.substr(2); - test['@id'] = - (manifest.baseIri || '') + - basename(manifest.filename).replace('.jsonld', '') + - test_id; - test.base = manifest.baseIri + test.input; - test.manifest = manifest; - const description = test_id + ' ' + (test.purpose || test.name); - - const _test = { - title: description, - f: makeFn({ - test, - run: ({test, testInfo, params}) => { - return jsonld[testInfo.fn](...params); - } - }) - }; - // 'only' based on test manifest - // 'skip' handled via skip() - if('only' in test) { - _test.only = test.only; - } - tests.push(_test); -} - -function makeFn({ - test, - adjustParams = p => p, - run, - ignoreResult = false -}) { - return async function() { - const self = this; - self.timeout(5000); - const testInfo = TEST_TYPES[getJsonLdTestType(test)]; - - // skip based on test manifest - if('skip' in test && test.skip) { - if(options.verboseSkip) { - console.log('Skipping test due to manifest:', - {id: test['@id'], name: test.name}); - } - self.skip(); - } - - // skip based on unknown test type - const testTypes = Object.keys(TEST_TYPES); - if(!isJsonLdType(test, testTypes)) { - if(options.verboseSkip) { - const type = [].concat( - getJsonLdValues(test, '@type'), - getJsonLdValues(test, 'type') - ); - console.log('Skipping test due to unknown type:', - {id: test['@id'], name: test.name, type}); - } - self.skip(); - } - - // skip based on test type - if(isJsonLdType(test, SKIP_TESTS)) { - if(options.verboseSkip) { - const type = [].concat( - getJsonLdValues(test, '@type'), - getJsonLdValues(test, 'type') - ); - console.log('Skipping test due to test type:', - {id: test['@id'], name: test.name, type}); - } - self.skip(); - } - - // skip based on type info - if(testInfo.skip && testInfo.skip.type) { - if(options.verboseSkip) { - console.log('Skipping test due to type info:', - {id: test['@id'], name: test.name}); - } - self.skip(); - } - - // skip based on id regex - if(testInfo.skip && testInfo.skip.idRegex) { - testInfo.skip.idRegex.forEach(function(re) { - if(re.test(test['@id'])) { - if(options.verboseSkip) { - console.log('Skipping test due to id:', - {id: test['@id']}); - } - self.skip(); - } - }); - } - - // skip based on description regex - if(testInfo.skip && testInfo.skip.descriptionRegex) { - testInfo.skip.descriptionRegex.forEach(function(re) { - if(re.test(description)) { - if(options.verboseSkip) { - console.log('Skipping test due to description:', - {id: test['@id'], name: test.name, description}); - } - self.skip(); - } - }); - } - - // Make expandContext absolute to the manifest - if(test.hasOwnProperty('option') && test.option.expandContext) { - test.option.expandContext = - prependBase(test.manifest.baseIri, test.option.expandContext); - } - - const testOptions = getJsonLdValues(test, 'option'); - // allow special handling in case of normative test failures - let normativeTest = true; - - testOptions.forEach(function(opt) { - const processingModes = getJsonLdValues(opt, 'processingMode'); - processingModes.forEach(function(pm) { - let skipModes = []; - if(testInfo.skip && testInfo.skip.processingMode) { - skipModes = testInfo.skip.processingMode; - } - if(skipModes.indexOf(pm) !== -1) { - if(options.verboseSkip) { - console.log('Skipping test due to processingMode:', - {id: test['@id'], name: test.name, processingMode: pm}); - } - self.skip(); - } - }); - }); - - testOptions.forEach(function(opt) { - const specVersions = getJsonLdValues(opt, 'specVersion'); - specVersions.forEach(function(sv) { - let skipVersions = []; - if(testInfo.skip && testInfo.skip.specVersion) { - skipVersions = testInfo.skip.specVersion; - } - if(skipVersions.indexOf(sv) !== -1) { - if(options.verboseSkip) { - console.log('Skipping test due to specVersion:', - {id: test['@id'], name: test.name, specVersion: sv}); - } - self.skip(); - } - }); - }); - - testOptions.forEach(function(opt) { - const normative = getJsonLdValues(opt, 'normative'); - normative.forEach(function(n) { - normativeTest = normativeTest && n; - }); - }); - - const params = adjustParams(testInfo.params.map(param => param(test))); - // resolve test data - const values = await Promise.all(params); - // copy used to check inputs do not change - const valuesOrig = klona(values); - let err; - let result; - // run and capture errors and results - try { - result = await run({test, testInfo, params: values}); - // check input not changed - assert.deepStrictEqual(valuesOrig, values); - } catch(e) { - err = e; - } - - try { - if(isJsonLdType(test, 'jld:NegativeEvaluationTest')) { - if(!ignoreResult) { - await compareExpectedError(test, err); - } - } else if(isJsonLdType(test, 'jld:PositiveEvaluationTest') || - isJsonLdType(test, 'rdfc:Urgna2012EvalTest') || - isJsonLdType(test, 'rdfc:Urdna2015EvalTest')) { - if(err) { - throw err; - } - if(!ignoreResult) { - await testInfo.compare(test, result); - } - } else if(isJsonLdType(test, 'jld:PositiveSyntaxTest')) { - // no checks - } else { - throw Error('Unknown test type: ' + test.type); - } - - let benchmarkResult = null; - if(options.benchmarkOptions) { - const result = await runBenchmark({ - test, - testInfo, - run, - params: testInfo.params.map(param => param(test, { - // pre-load params to avoid doc loader and parser timing - load: true - })), - mochaTest: self - }); - benchmarkResult = { - // FIXME use generic prefix - '@type': 'jldb:BenchmarkResult', - 'jldb:hz': result.target.hz, - 'jldb:rme': result.target.stats.rme - }; - } - - if(options.earl.report) { - options.earl.report.addAssertion(test, true, { - benchmarkResult - }); - } - } catch(err) { - // FIXME: improve handling of non-normative errors - // FIXME: for now, explicitly disabling tests. - //if(!normativeTest) { - // // failure ok - // if(options.verboseSkip) { - // console.log('Skipping non-normative test due to failure:', - // {id: test['@id'], name: test.name}); - // } - // self.skip(); - //} - if(options.bailOnError) { - if(err.name !== 'AssertionError') { - console.error('\nError: ', JSON.stringify(err, null, 2)); - } - options.exit(); - } - if(options.earl.report) { - options.earl.report.addAssertion(test, false); - } - console.error('Error: ', JSON.stringify(err, null, 2)); - throw err; - } - }; -} - -async function runBenchmark({test, testInfo, params, run, mochaTest}) { - const values = await Promise.all(params); - - return new Promise((resolve, reject) => { - const suite = new benchmark.Suite(); - suite.add({ - name: test.name, - defer: true, - fn: deferred => { - run({test, testInfo, params: values}).then(() => { - deferred.resolve(); - }); - } - }); - suite - .on('start', e => { - // set timeout to a bit more than max benchmark time - mochaTest.timeout((e.target.maxTime + 2) * 1000); - }) - .on('cycle', e => { - console.log(String(e.target)); - }) - .on('error', err => { - reject(new Error(err)); - }) - .on('complete', e => { - resolve(e); - }) - .run({async: true}); - }); -} - -function getJsonLdTestType(test) { - const types = Object.keys(TEST_TYPES); - for(let i = 0; i < types.length; ++i) { - if(isJsonLdType(test, types[i])) { - return types[i]; - } - } - return null; -} - -function readManifestEntry(manifest, entry) { - let p = Promise.resolve(); - let _entry = entry; - if(typeof entry === 'string') { - let _filename; - p = p.then(() => { - if(entry.endsWith('json') || entry.endsWith('jsonld')) { - // load as file - return entry; - } - // load as dir with manifest.jsonld - return joinPath(entry, 'manifest.jsonld'); - }).then(entry => { - const dir = dirname(manifest.filename); - return joinPath(dir, entry); - }).then(filename => { - _filename = filename; - return readJson(filename); - }).then(entry => { - _entry = entry; - _entry.filename = _filename; - return _entry; - }).catch(err => { - if(err.code === 'ENOENT') { - //console.log('File does not exist, skipping: ' + _filename); - // return a "skip" entry - _entry = { - type: '__SKIP__', - title: 'Not found, skipping: ' + _filename, - filename: _filename, - skip: true - }; - return; - } - throw err; - }); - } - return p.then(() => { - _entry.dirname = dirname(_entry.filename || manifest.filename); - return _entry; - }); -} - -function readTestUrl(property) { - return async function(test, options) { - if(!test[property]) { - return null; - } - if(options && options.load) { - // always load - const filename = await joinPath(test.dirname, test[property]); - return readJson(filename); - } - return test.manifest.baseIri + test[property]; - }; -} - -function readTestJson(property) { - return async function(test) { - if(!test[property]) { - return null; - } - const filename = await joinPath(test.dirname, test[property]); - return readJson(filename); - }; -} - -function readTestNQuads(property) { - return async function(test) { - if(!test[property]) { - return null; - } - const filename = await joinPath(test.dirname, test[property]); - return readFile(filename); - }; -} - -function createTestOptions(opts) { - return function(test) { - const options = { - documentLoader: createDocumentLoader(test) - }; - const httpOptions = ['contentType', 'httpLink', 'httpStatus', 'redirectTo']; - const testOptions = test.option || {}; - for(const key in testOptions) { - if(httpOptions.indexOf(key) === -1) { - options[key] = testOptions[key]; - } - } - if(opts) { - // extend options - for(const key in opts) { - options[key] = opts[key]; - } - } - return options; - }; -} - -// find the expected output property or throw error -function _getExpectProperty(test) { - if('expectErrorCode' in test) { - return 'expectErrorCode'; - } else if('expect' in test) { - return 'expect'; - } else if('result' in test) { - return 'result'; - } else { - throw Error('No expected output property found'); - } -} - -async function compareExpectedJson(test, result) { - let expect; - try { - expect = await readTestJson(_getExpectProperty(test))(test); - assert.deepStrictEqual(result, expect); - } catch(err) { - if(options.bailOnError) { - console.log('\nTEST FAILED\n'); - console.log('EXPECTED: ' + JSON.stringify(expect, null, 2)); - console.log('ACTUAL: ' + JSON.stringify(result, null, 2)); - } - throw err; - } -} - -async function compareExpectedNQuads(test, result) { - let expect; - try { - expect = await readTestNQuads(_getExpectProperty(test))(test); - assert.strictEqual(result, expect); - } catch(ex) { - if(options.bailOnError) { - console.log('\nTEST FAILED\n'); - console.log('EXPECTED:\n' + expect); - console.log('ACTUAL:\n' + result); - } - throw ex; - } -} - -async function compareCanonizedExpectedNQuads(test, result) { - let expect; - try { - expect = await readTestNQuads(_getExpectProperty(test))(test); - const opts = {algorithm: 'URDNA2015'}; - const expectDataset = rdfCanonize.NQuads.parse(expect); - const expectCmp = await rdfCanonize.canonize(expectDataset, opts); - const resultDataset = rdfCanonize.NQuads.parse(result); - const resultCmp = await rdfCanonize.canonize(resultDataset, opts); - assert.strictEqual(resultCmp, expectCmp); - } catch(err) { - if(options.bailOnError) { - console.log('\nTEST FAILED\n'); - console.log('EXPECTED:\n' + expect); - console.log('ACTUAL:\n' + result); - } - throw err; - } -} - -async function compareExpectedError(test, err) { - let expect; - let result; - try { - expect = test[_getExpectProperty(test)]; - result = getJsonLdErrorCode(err); - assert.ok(err, 'no error present'); - assert.strictEqual(result, expect); - } catch(_err) { - if(options.bailOnError) { - console.log('\nTEST FAILED\n'); - console.log('EXPECTED: ' + expect); - console.log('ACTUAL: ' + result); - } - // log the unexpected error to help with debugging - console.log('Unexpected error:', err); - throw _err; - } -} - -function isJsonLdType(node, type) { - const nodeType = [].concat( - getJsonLdValues(node, '@type'), - getJsonLdValues(node, 'type') - ); - type = Array.isArray(type) ? type : [type]; - for(let i = 0; i < type.length; ++i) { - if(nodeType.indexOf(type[i]) !== -1) { - return true; - } - } - return false; -} - -function getJsonLdValues(node, property) { - let rval = []; - if(property in node) { - rval = node[property]; - if(!Array.isArray(rval)) { - rval = [rval]; - } - } - return rval; -} - -function getJsonLdErrorCode(err) { - if(!err) { - return null; - } - if(err.details) { - if(err.details.code) { - return err.details.code; - } - if(err.details.cause) { - return getJsonLdErrorCode(err.details.cause); - } - } - return err.name; -} - -async function readJson(filename) { - const data = await readFile(filename); - return JSON.parse(data); -} - -async function readFile(filename) { - return options.readFile(filename); -} - -async function joinPath() { - return join.apply(null, Array.prototype.slice.call(arguments)); -} - -function dirname(filename) { - if(options.nodejs) { - return options.nodejs.path.dirname(filename); - } - const idx = filename.lastIndexOf('/'); - if(idx === -1) { - return filename; - } - return filename.substr(0, idx); -} - -function basename(filename) { - if(options.nodejs) { - return options.nodejs.path.basename(filename); - } - const idx = filename.lastIndexOf('/'); - if(idx === -1) { - return filename; - } - return filename.substr(idx + 1); -} - -// check test.option.loader.rewrite map for url, -// if no test rewrite, check manifest, -// else no rewrite -function rewrite(test, url) { - if(test.option && - test.option.loader && - test.option.loader.rewrite && - url in test.option.loader.rewrite) { - return test.option.loader.rewrite[url]; - } - const manifest = test.manifest; - if(manifest.option && - manifest.option.loader && - manifest.option.loader.rewrite && - url in manifest.option.loader.rewrite) { - return manifest.option.loader.rewrite[url]; - } - return url; -} - -/** - * Creates a test remote document loader. - * - * @param test the test to use the document loader for. - * - * @return the document loader. - */ -function createDocumentLoader(test) { - const localBases = [ - 'http://json-ld.org/test-suite', - 'https://json-ld.org/test-suite', - 'https://json-ld.org/benchmarks', - 'https://w3c.github.io/json-ld-api/tests', - 'https://w3c.github.io/json-ld-framing/tests' - ]; - - const localLoader = function(url) { - // always load remote-doc tests remotely in node - // NOTE: disabled due to github pages issues. - //if(options.nodejs && test.manifest.name === 'Remote document') { - // return jsonld.documentLoader(url); - //} - - // handle loader rewrite options for test or manifest - url = rewrite(test, url); - - // FIXME: this check only works for main test suite and will not work if: - // - running other tests and main test suite not installed - // - use other absolute URIs but want to load local files - const isTestSuite = localBases.some(function(base) { - return url.startsWith(base); - }); - // TODO: improve this check - const isRelative = url.indexOf(':') === -1; - if(isTestSuite || isRelative) { - // attempt to load official test-suite files or relative URLs locally - return loadLocally(url); - } - - // load remotely - return jsonld.documentLoader(url); - }; - - return localLoader; - - function loadLocally(url) { - const doc = {contextUrl: null, documentUrl: url, document: null}; - const options = test.option; - if(options && url === test.base) { - if('redirectTo' in options && parseInt(options.httpStatus, 10) >= 300) { - doc.documentUrl = test.manifest.baseIri + options.redirectTo; - } else if('httpLink' in options) { - let contentType = options.contentType || null; - if(!contentType && url.indexOf('.jsonld', url.length - 7) !== -1) { - contentType = 'application/ld+json'; - } - if(!contentType && url.indexOf('.json', url.length - 5) !== -1) { - contentType = 'application/json'; - } - let linkHeader = options.httpLink; - if(Array.isArray(linkHeader)) { - linkHeader = linkHeader.join(','); - } - const linkHeaders = jsonld.parseLinkHeader(linkHeader); - const linkedContext = - linkHeaders['http://www.w3.org/ns/json-ld#context']; - if(linkedContext && contentType !== 'application/ld+json') { - if(Array.isArray(linkedContext)) { - throw {name: 'multiple context link headers'}; - } - doc.contextUrl = linkedContext.target; - } - - // If not JSON-LD, alternate may point there - if(linkHeaders.alternate && - linkHeaders.alternate.type == 'application/ld+json' && - !(contentType || '').match(/^application\/(\w*\+)?json$/)) { - doc.documentUrl = prependBase(url, linkHeaders.alternate.target); - } - } - } - - let p = Promise.resolve(); - if(doc.documentUrl.indexOf(':') === -1) { - p = p.then(() => { - return joinPath(test.manifest.dirname, doc.documentUrl); - }).then(filename => { - doc.documentUrl = 'file://' + filename; - return filename; - }); - } else { - p = p.then(() => { - return joinPath( - test.manifest.dirname, - doc.documentUrl.substr(test.manifest.baseIri.length)); - }).then(fn => { - return fn; - }); - } - - return p.then(readJson).then(json => { - doc.document = json; - return doc; - }).catch(() => { - throw {name: 'loading document failed', url}; - }); - } -} - -}; diff --git a/tests/test-karma.js b/tests/test-karma.js index 33910732..5d04e8ec 100644 --- a/tests/test-karma.js +++ b/tests/test-karma.js @@ -46,7 +46,7 @@ mocha.setup({delay: true, ui: 'bdd'}); const assert = require('chai').assert; -const common = require('./test-common'); +const common = require('./test'); const jsonld = require('..'); const server = require('karma-server-side'); const webidl = require('./test-webidl'); diff --git a/tests/test-node.js b/tests/test-node.js new file mode 100644 index 00000000..c9001aa8 --- /dev/null +++ b/tests/test-node.js @@ -0,0 +1,213 @@ +/** + * Node.js test runner for jsonld.js. + * + * Use environment vars to control: + * + * Set dirs, manifests, or js to run: + * JSONLD_TESTS="r1 r2 ..." + * Output an EARL report: + * EARL=filename + * Test environment details for EARL report: + * This is useful for benchmark comparison. + * By default no details are added for privacy reasons. + * Automatic details can be added for all fields with '1', 'true', or 'auto': + * TEST_ENV=1 + * To include only certain fields, set them, or use 'auto': + * TEST_ENV=cpu='Intel i7-4790K @ 4.00GHz',runtime='Node.js',... + * TEST_ENV=cpu=auto # only cpu + * TEST_ENV=cpu,runtime # only cpu and runtime + * TEST_ENV=auto,comment='special test' # all auto with override + * Available fields: + * - label - ex: 'Setup 1' (short label for reports) + * - arch - ex: 'x64' + * - cpu - ex: 'Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz' + * - cpuCount - ex: 8 + * - platform - ex: 'linux' + * - runtime - ex: 'Node.js' + * - runtimeVersion - ex: 'v14.19.0' + * - comment: any text + * - version: jsonld.js version + * Bail with tests fail: + * BAIL=true + * Verbose skip reasons: + * VERBOSE_SKIP=true + * Benchmark mode: + * Basic: + * JSONLD_BENCHMARK=1 + * With options: + * JSONLD_BENCHMARK=key1=value1,key2=value2,... + * + * @author Dave Longley + * @author David I. Lehn + * + * Copyright (c) 2011-2022 Digital Bazaar, Inc. All rights reserved. + */ +const assert = require('chai').assert; +const benchmark = require('benchmark'); +const common = require('./test'); +const fs = require('fs-extra'); +const jsonld = require('..'); +const os = require('os'); +const path = require('path'); + +const entries = []; + +if(process.env.JSONLD_TESTS) { + entries.push(...process.env.JSONLD_TESTS.split(' ')); +} else { + const _top = path.resolve(__dirname, '..'); + + // json-ld-api main test suite + const apiPath = path.resolve(_top, 'test-suites/json-ld-api/tests'); + if(fs.existsSync(apiPath)) { + entries.push(apiPath); + } else { + // default to sibling dir + entries.push(path.resolve(_top, '../json-ld-api/tests')); + } + + // json-ld-framing main test suite + const framingPath = path.resolve(_top, 'test-suites/json-ld-framing/tests'); + if(fs.existsSync(framingPath)) { + entries.push(framingPath); + } else { + // default to sibling dir + entries.push(path.resolve(_top, '../json-ld-framing/tests')); + } + + /* + // TODO: use json-ld-framing once tests are moved + // json-ld.org framing test suite + const framingPath = path.resolve( + _top, 'test-suites/json-ld.org/test-suite/tests/frame-manifest.jsonld'); + if(fs.existsSync(framingPath)) { + entries.push(framingPath); + } else { + // default to sibling dir + entries.push(path.resolve( + _top, '../json-ld.org/test-suite/tests/frame-manifest.jsonld')); + } + */ + + // W3C RDF Dataset Canonicalization "rdf-canon" test suite + const rdfCanonPath = path.resolve(_top, 'test-suites/rdf-canon/tests'); + if(fs.existsSync(rdfCanonPath)) { + entries.push(rdfCanonPath); + } else { + // default up to sibling dir + entries.push(path.resolve(_top, '../rdf-canon/tests')); + } + + // other tests + entries.push(path.resolve(_top, 'tests/misc.js')); + entries.push(path.resolve(_top, 'tests/graph-container.js')); + entries.push(path.resolve(_top, 'tests/node-document-loader-tests.js')); +} + +// test environment +let testEnv = null; +if(process.env.TEST_ENV) { + let _test_env = process.env.TEST_ENV; + if(!(['0', 'false'].includes(_test_env))) { + testEnv = {}; + if(['1', 'true', 'auto'].includes(_test_env)) { + _test_env = 'auto'; + } + _test_env.split(',').forEach(pair => { + if(pair === 'auto') { + testEnv.name = 'auto'; + testEnv.arch = 'auto'; + testEnv.cpu = 'auto'; + testEnv.cpuCount = 'auto'; + testEnv.platform = 'auto'; + testEnv.runtime = 'auto'; + testEnv.runtimeVersion = 'auto'; + testEnv.comment = 'auto'; + testEnv.version = 'auto'; + } else { + const kv = pair.split('='); + if(kv.length === 1) { + testEnv[kv[0]] = 'auto'; + } else { + testEnv[kv[0]] = kv.slice(1).join('='); + } + } + }); + if(testEnv.label === 'auto') { + testEnv.label = ''; + } + if(testEnv.arch === 'auto') { + testEnv.arch = process.arch; + } + if(testEnv.cpu === 'auto') { + testEnv.cpu = os.cpus()[0].model; + } + if(testEnv.cpuCount === 'auto') { + testEnv.cpuCount = os.cpus().length; + } + if(testEnv.platform === 'auto') { + testEnv.platform = process.platform; + } + if(testEnv.runtime === 'auto') { + testEnv.runtime = 'Node.js'; + } + if(testEnv.runtimeVersion === 'auto') { + testEnv.runtimeVersion = process.version; + } + if(testEnv.comment === 'auto') { + testEnv.comment = ''; + } + if(testEnv.version === 'auto') { + testEnv.version = require('../package.json').version; + } + } +} + +let benchmarkOptions = null; +if(process.env.JSONLD_BENCHMARK) { + if(!(['0', 'false'].includes(process.env.JSONLD_BENCHMARK))) { + benchmarkOptions = {}; + if(!(['1', 'true'].includes(process.env.JSONLD_BENCHMARK))) { + process.env.JSONLD_BENCHMARK.split(',').forEach(pair => { + const kv = pair.split('='); + benchmarkOptions[kv[0]] = kv[1]; + }); + } + } +} + +const options = { + nodejs: { + path + }, + assert, + benchmark, + jsonld, + exit: code => process.exit(code), + earl: { + filename: process.env.EARL + }, + verboseSkip: process.env.VERBOSE_SKIP === 'true', + bailOnError: process.env.BAIL === 'true', + entries, + testEnv, + benchmarkOptions, + readFile: filename => { + return fs.readFile(filename, 'utf8'); + }, + writeFile: (filename, data) => { + return fs.outputFile(filename, data); + }, + import: f => require(f) +}; + +// wait for setup of all tests then run mocha +common(options).then(() => { + run(); +}).catch(err => { + console.error(err); +}); + +process.on('unhandledRejection', (reason, p) => { + console.error('Unhandled Rejection at:', p, 'reason:', reason); +}); diff --git a/tests/test.js b/tests/test.js index 074161d2..500fe9ed 100644 --- a/tests/test.js +++ b/tests/test.js @@ -1,213 +1,1113 @@ /** - * Node.js test runner for jsonld.js. - * - * Use environment vars to control: - * - * Set dirs, manifests, or js to run: - * JSONLD_TESTS="r1 r2 ..." - * Output an EARL report: - * EARL=filename - * Test environment details for EARL report: - * This is useful for benchmark comparison. - * By default no details are added for privacy reasons. - * Automatic details can be added for all fields with '1', 'true', or 'auto': - * TEST_ENV=1 - * To include only certain fields, set them, or use 'auto': - * TEST_ENV=cpu='Intel i7-4790K @ 4.00GHz',runtime='Node.js',... - * TEST_ENV=cpu=auto # only cpu - * TEST_ENV=cpu,runtime # only cpu and runtime - * TEST_ENV=auto,comment='special test' # all auto with override - * Available fields: - * - label - ex: 'Setup 1' (short label for reports) - * - arch - ex: 'x64' - * - cpu - ex: 'Intel(R) Core(TM) i7-4790K CPU @ 4.00GHz' - * - cpuCount - ex: 8 - * - platform - ex: 'linux' - * - runtime - ex: 'Node.js' - * - runtimeVersion - ex: 'v14.19.0' - * - comment: any text - * - version: jsonld.js version - * Bail with tests fail: - * BAIL=true - * Verbose skip reasons: - * VERBOSE_SKIP=true - * Benchmark mode: - * Basic: - * JSONLD_BENCHMARK=1 - * With options: - * JSONLD_BENCHMARK=key1=value1,key2=value2,... - * - * @author Dave Longley - * @author David I. Lehn - * - * Copyright (c) 2011-2022 Digital Bazaar, Inc. All rights reserved. + * Copyright (c) 2011-2019 Digital Bazaar, Inc. All rights reserved. */ -const assert = require('chai').assert; -const benchmark = require('benchmark'); -const common = require('./test-common'); -const fs = require('fs-extra'); -const jsonld = require('..'); -const os = require('os'); -const path = require('path'); - -const entries = []; - -if(process.env.JSONLD_TESTS) { - entries.push(...process.env.JSONLD_TESTS.split(' ')); -} else { - const _top = path.resolve(__dirname, '..'); - - // json-ld-api main test suite - const apiPath = path.resolve(_top, 'test-suites/json-ld-api/tests'); - if(fs.existsSync(apiPath)) { - entries.push(apiPath); - } else { - // default to sibling dir - entries.push(path.resolve(_top, '../json-ld-api/tests')); +/* eslint-disable indent */ +const EarlReport = require('./earl-report'); +const join = require('join-path-js'); +const rdfCanonize = require('rdf-canonize'); +const {prependBase} = require('../lib/url'); +const {klona} = require('klona'); + +module.exports = function(options) { + +'use strict'; + +const assert = options.assert; +const benchmark = options.benchmark; +const jsonld = options.jsonld; + +const manifest = options.manifest || { + '@context': 'https://json-ld.org/test-suite/context.jsonld', + '@id': '', + '@type': 'mf:Manifest', + description: 'Top level jsonld.js manifest', + name: 'jsonld.js', + sequence: options.entries || [], + filename: '/' +}; + +const TEST_TYPES = { + 'jld:CompactTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + /compact-manifest#t0112$/, + /compact-manifest#t0113$/, + // html + /html-manifest#tc001$/, + /html-manifest#tc002$/, + /html-manifest#tc003$/, + /html-manifest#tc004$/, + ] + }, + fn: 'compact', + params: [ + readTestUrl('input'), + readTestJson('context'), + createTestOptions() + ], + compare: compareExpectedJson + }, + 'jld:ExpandTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + // spec issues + // Unclear how to handle {"@id": null} edge case + // See https://github.com/w3c/json-ld-api/issues/480 + // non-normative test, also see toRdf-manifest#te122 + ///expand-manifest#t0122$/, + + // misc + /expand-manifest#tc037$/, + /expand-manifest#tc038$/, + /expand-manifest#ter54$/, + + // html + /html-manifest#te001$/, + /html-manifest#te002$/, + /html-manifest#te003$/, + /html-manifest#te004$/, + /html-manifest#te005$/, + /html-manifest#te006$/, + /html-manifest#te007$/, + /html-manifest#te010$/, + /html-manifest#te011$/, + /html-manifest#te012$/, + /html-manifest#te013$/, + /html-manifest#te014$/, + /html-manifest#te015$/, + /html-manifest#te016$/, + /html-manifest#te017$/, + /html-manifest#te018$/, + /html-manifest#te019$/, + /html-manifest#te020$/, + /html-manifest#te021$/, + /html-manifest#te022$/, + /html-manifest#tex01$/, + // HTML extraction + /expand-manifest#thc01$/, + /expand-manifest#thc02$/, + /expand-manifest#thc03$/, + /expand-manifest#thc04$/, + /expand-manifest#thc05$/, + // remote + /remote-doc-manifest#t0013$/, // HTML + ] + }, + fn: 'expand', + params: [ + readTestUrl('input'), + createTestOptions() + ], + compare: compareExpectedJson + }, + 'jld:FlattenTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + // html + /html-manifest#tf001$/, + /html-manifest#tf002$/, + /html-manifest#tf003$/, + /html-manifest#tf004$/, + ] + }, + fn: 'flatten', + params: [ + readTestUrl('input'), + readTestJson('context'), + createTestOptions() + ], + compare: compareExpectedJson + }, + 'jld:FrameTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + /frame-manifest#t0069$/, + ] + }, + fn: 'frame', + params: [ + readTestUrl('input'), + readTestJson('frame'), + createTestOptions() + ], + compare: compareExpectedJson + }, + 'jld:FromRDFTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + // direction (compound-literal) + /fromRdf-manifest#tdi11$/, + /fromRdf-manifest#tdi12$/, + ] + }, + fn: 'fromRDF', + params: [ + readTestNQuads('input'), + createTestOptions({format: 'application/n-quads'}) + ], + compare: compareExpectedJson + }, + 'jld:NormalizeTest': { + fn: 'normalize', + params: [ + readTestUrl('input'), + createTestOptions({format: 'application/n-quads'}) + ], + compare: compareExpectedNQuads + }, + 'jld:ToRDFTest': { + skip: { + // skip tests where behavior changed for a 1.1 processor + // see JSON-LD 1.0 Errata + specVersion: ['json-ld-1.0'], + // FIXME + // NOTE: idRegex format: + //MMM-manifest#tNNN$/, + idRegex: [ + // spec issues + // Unclear how to handle {"@id": null} edge case + // See https://github.com/w3c/json-ld-api/issues/480 + // normative test, also see expand-manifest#t0122 + ///toRdf-manifest#te122$/, + + // misc + /toRdf-manifest#tc037$/, + /toRdf-manifest#tc038$/, + /toRdf-manifest#ter54$/, + /toRdf-manifest#tli12$/, + /toRdf-manifest#tli14$/, + + // well formed + /toRdf-manifest#twf05$/, + + // html + /html-manifest#tr001$/, + /html-manifest#tr002$/, + /html-manifest#tr003$/, + /html-manifest#tr004$/, + /html-manifest#tr005$/, + /html-manifest#tr006$/, + /html-manifest#tr007$/, + /html-manifest#tr010$/, + /html-manifest#tr011$/, + /html-manifest#tr012$/, + /html-manifest#tr013$/, + /html-manifest#tr014$/, + /html-manifest#tr015$/, + /html-manifest#tr016$/, + /html-manifest#tr017$/, + /html-manifest#tr018$/, + /html-manifest#tr019$/, + /html-manifest#tr020$/, + /html-manifest#tr021$/, + /html-manifest#tr022$/, + // Invalid Statement + /toRdf-manifest#te075$/, + /toRdf-manifest#te111$/, + /toRdf-manifest#te112$/, + // direction (compound-literal) + /toRdf-manifest#tdi11$/, + /toRdf-manifest#tdi12$/, + ] + }, + fn: 'toRDF', + params: [ + readTestUrl('input'), + createTestOptions({format: 'application/n-quads'}) + ], + compare: compareCanonizedExpectedNQuads + }, + 'rdfc:Urgna2012EvalTest': { + fn: 'normalize', + params: [ + readTestNQuads('action'), + createTestOptions({ + algorithm: 'URGNA2012', + inputFormat: 'application/n-quads', + format: 'application/n-quads' + }) + ], + compare: compareExpectedNQuads + }, + 'rdfc:Urdna2015EvalTest': { + skip: { + // NOTE: idRegex format: + //manifest-urdna2015#testNNN$/, + idRegex: [ + // Unsupported U escape + /manifest-urdna2015#test060/ + ] + }, + fn: 'normalize', + params: [ + readTestNQuads('action'), + createTestOptions({ + algorithm: 'URDNA2015', + inputFormat: 'application/n-quads', + format: 'application/n-quads' + }) + ], + compare: compareExpectedNQuads } +}; - // json-ld-framing main test suite - const framingPath = path.resolve(_top, 'test-suites/json-ld-framing/tests'); - if(fs.existsSync(framingPath)) { - entries.push(framingPath); - } else { - // default to sibling dir - entries.push(path.resolve(_top, '../json-ld-framing/tests')); +const SKIP_TESTS = []; + +// create earl report +if(options.earl && options.earl.filename) { + options.earl.report = new EarlReport({ + env: options.testEnv + }); + if(options.benchmarkOptions) { + options.earl.report.setupForBenchmarks({testEnv: options.testEnv}); } +} - /* - // TODO: use json-ld-framing once tests are moved - // json-ld.org framing test suite - const framingPath = path.resolve( - _top, 'test-suites/json-ld.org/test-suite/tests/frame-manifest.jsonld'); - if(fs.existsSync(framingPath)) { - entries.push(framingPath); - } else { - // default to sibling dir - entries.push(path.resolve( - _top, '../json-ld.org/test-suite/tests/frame-manifest.jsonld')); +return new Promise(resolve => { + +// async generated tests +// _tests => [{suite}, ...] +// suite => { +// title: ..., +// tests: [test, ...], +// suites: [suite, ...] +// } +const _tests = []; + +return addManifest(manifest, _tests) + .then(() => { + return _testsToMocha(_tests); + }).then(result => { + if(options.earl.report) { + describe('Writing EARL report to: ' + options.earl.filename, function() { + // print out EARL even if .only was used + const _it = result.hadOnly ? it.only : it; + _it('should print the earl report', function() { + return options.writeFile( + options.earl.filename, options.earl.report.reportJson()); + }); + }); + } + }).then(() => resolve()); + +// build mocha tests from local test structure +function _testsToMocha(tests) { + let hadOnly = false; + tests.forEach(suite => { + if(suite.skip) { + describe.skip(suite.title); + return; + } + describe(suite.title, () => { + suite.tests.forEach(test => { + if(test.only) { + hadOnly = true; + it.only(test.title, test.f); + return; + } + it(test.title, test.f); + }); + const {hadOnly: _hadOnly} = _testsToMocha(suite.suites); + hadOnly = hadOnly || _hadOnly; + }); + suite.imports.forEach(f => { + options.import(f); + }); + }); + return { + hadOnly + }; +} + +}); + +/** + * Adds the tests for all entries in the given manifest. + * + * @param manifest {Object} the manifest. + * @param parent {Object} the parent test structure + * @return {Promise} + */ +function addManifest(manifest, parent) { + return new Promise((resolve, reject) => { + // create test structure + const suite = { + title: manifest.name || manifest.label, + tests: [], + suites: [], + imports: [] + }; + parent.push(suite); + + // get entries and sequence (alias for entries) + const entries = [].concat( + getJsonLdValues(manifest, 'entries'), + getJsonLdValues(manifest, 'sequence') + ); + + const includes = getJsonLdValues(manifest, 'include'); + // add includes to sequence as jsonld files + for(let i = 0; i < includes.length; ++i) { + entries.push(includes[i] + '.jsonld'); + } + + // resolve all entry promises and process + Promise.all(entries).then(entries => { + let p = Promise.resolve(); + entries.forEach(entry => { + if(typeof entry === 'string' && entry.endsWith('js')) { + // process later as a plain JavaScript file + suite.imports.push(entry); + return; + } else if(typeof entry === 'function') { + // process as a function that returns a promise + p = p.then(() => { + return entry(options); + }).then(childSuite => { + if(suite) { + suite.suites.push(childSuite); + } + }); + return; + } + p = p.then(() => { + return readManifestEntry(manifest, entry); + }).then(entry => { + if(isJsonLdType(entry, '__SKIP__')) { + // special local skip logic + suite.tests.push(entry); + } else if(isJsonLdType(entry, 'mf:Manifest')) { + // entry is another manifest + return addManifest(entry, suite.suites); + } else { + // assume entry is a test + return addTest(manifest, entry, suite.tests); + } + }); + }); + return p; + }).then(() => { + resolve(); + }).catch(err => { + console.error(err); + reject(err); + }); + }); +} + +/** + * Adds a test. + * + * @param manifest {Object} the manifest. + * @param test {Object} the test. + * @param tests {Array} the list of tests to add to. + * @return {Promise} + */ +async function addTest(manifest, test, tests) { + // expand @id and input base + const test_id = test['@id'] || test.id; + //var number = test_id.substr(2); + test['@id'] = + (manifest.baseIri || '') + + basename(manifest.filename).replace('.jsonld', '') + + test_id; + test.base = manifest.baseIri + test.input; + test.manifest = manifest; + const description = test_id + ' ' + (test.purpose || test.name); + + const _test = { + title: description, + f: makeFn({ + test, + run: ({test, testInfo, params}) => { + return jsonld[testInfo.fn](...params); + } + }) + }; + // 'only' based on test manifest + // 'skip' handled via skip() + if('only' in test) { + _test.only = test.only; } - */ + tests.push(_test); +} - // W3C RDF Dataset Canonicalization "rdf-canon" test suite - const rdfCanonPath = path.resolve(_top, 'test-suites/rdf-canon/tests'); - if(fs.existsSync(rdfCanonPath)) { - entries.push(rdfCanonPath); - } else { - // default up to sibling dir - entries.push(path.resolve(_top, '../rdf-canon/tests')); - } - - // other tests - entries.push(path.resolve(_top, 'tests/misc.js')); - entries.push(path.resolve(_top, 'tests/graph-container.js')); - entries.push(path.resolve(_top, 'tests/node-document-loader-tests.js')); -} - -// test environment -let testEnv = null; -if(process.env.TEST_ENV) { - let _test_env = process.env.TEST_ENV; - if(!(['0', 'false'].includes(_test_env))) { - testEnv = {}; - if(['1', 'true', 'auto'].includes(_test_env)) { - _test_env = 'auto'; - } - _test_env.split(',').forEach(pair => { - if(pair === 'auto') { - testEnv.name = 'auto'; - testEnv.arch = 'auto'; - testEnv.cpu = 'auto'; - testEnv.cpuCount = 'auto'; - testEnv.platform = 'auto'; - testEnv.runtime = 'auto'; - testEnv.runtimeVersion = 'auto'; - testEnv.comment = 'auto'; - testEnv.version = 'auto'; +function makeFn({ + test, + adjustParams = p => p, + run, + ignoreResult = false +}) { + return async function() { + const self = this; + self.timeout(5000); + const testInfo = TEST_TYPES[getJsonLdTestType(test)]; + + // skip based on test manifest + if('skip' in test && test.skip) { + if(options.verboseSkip) { + console.log('Skipping test due to manifest:', + {id: test['@id'], name: test.name}); + } + self.skip(); + } + + // skip based on unknown test type + const testTypes = Object.keys(TEST_TYPES); + if(!isJsonLdType(test, testTypes)) { + if(options.verboseSkip) { + const type = [].concat( + getJsonLdValues(test, '@type'), + getJsonLdValues(test, 'type') + ); + console.log('Skipping test due to unknown type:', + {id: test['@id'], name: test.name, type}); + } + self.skip(); + } + + // skip based on test type + if(isJsonLdType(test, SKIP_TESTS)) { + if(options.verboseSkip) { + const type = [].concat( + getJsonLdValues(test, '@type'), + getJsonLdValues(test, 'type') + ); + console.log('Skipping test due to test type:', + {id: test['@id'], name: test.name, type}); + } + self.skip(); + } + + // skip based on type info + if(testInfo.skip && testInfo.skip.type) { + if(options.verboseSkip) { + console.log('Skipping test due to type info:', + {id: test['@id'], name: test.name}); + } + self.skip(); + } + + // skip based on id regex + if(testInfo.skip && testInfo.skip.idRegex) { + testInfo.skip.idRegex.forEach(function(re) { + if(re.test(test['@id'])) { + if(options.verboseSkip) { + console.log('Skipping test due to id:', + {id: test['@id']}); + } + self.skip(); + } + }); + } + + // skip based on description regex + if(testInfo.skip && testInfo.skip.descriptionRegex) { + testInfo.skip.descriptionRegex.forEach(function(re) { + if(re.test(description)) { + if(options.verboseSkip) { + console.log('Skipping test due to description:', + {id: test['@id'], name: test.name, description}); + } + self.skip(); + } + }); + } + + // Make expandContext absolute to the manifest + if(test.hasOwnProperty('option') && test.option.expandContext) { + test.option.expandContext = + prependBase(test.manifest.baseIri, test.option.expandContext); + } + + const testOptions = getJsonLdValues(test, 'option'); + // allow special handling in case of normative test failures + let normativeTest = true; + + testOptions.forEach(function(opt) { + const processingModes = getJsonLdValues(opt, 'processingMode'); + processingModes.forEach(function(pm) { + let skipModes = []; + if(testInfo.skip && testInfo.skip.processingMode) { + skipModes = testInfo.skip.processingMode; + } + if(skipModes.indexOf(pm) !== -1) { + if(options.verboseSkip) { + console.log('Skipping test due to processingMode:', + {id: test['@id'], name: test.name, processingMode: pm}); + } + self.skip(); + } + }); + }); + + testOptions.forEach(function(opt) { + const specVersions = getJsonLdValues(opt, 'specVersion'); + specVersions.forEach(function(sv) { + let skipVersions = []; + if(testInfo.skip && testInfo.skip.specVersion) { + skipVersions = testInfo.skip.specVersion; + } + if(skipVersions.indexOf(sv) !== -1) { + if(options.verboseSkip) { + console.log('Skipping test due to specVersion:', + {id: test['@id'], name: test.name, specVersion: sv}); + } + self.skip(); + } + }); + }); + + testOptions.forEach(function(opt) { + const normative = getJsonLdValues(opt, 'normative'); + normative.forEach(function(n) { + normativeTest = normativeTest && n; + }); + }); + + const params = adjustParams(testInfo.params.map(param => param(test))); + // resolve test data + const values = await Promise.all(params); + // copy used to check inputs do not change + const valuesOrig = klona(values); + let err; + let result; + // run and capture errors and results + try { + result = await run({test, testInfo, params: values}); + // check input not changed + assert.deepStrictEqual(valuesOrig, values); + } catch(e) { + err = e; + } + + try { + if(isJsonLdType(test, 'jld:NegativeEvaluationTest')) { + if(!ignoreResult) { + await compareExpectedError(test, err); + } + } else if(isJsonLdType(test, 'jld:PositiveEvaluationTest') || + isJsonLdType(test, 'rdfc:Urgna2012EvalTest') || + isJsonLdType(test, 'rdfc:Urdna2015EvalTest')) { + if(err) { + throw err; + } + if(!ignoreResult) { + await testInfo.compare(test, result); + } + } else if(isJsonLdType(test, 'jld:PositiveSyntaxTest')) { + // no checks } else { - const kv = pair.split('='); - if(kv.length === 1) { - testEnv[kv[0]] = 'auto'; - } else { - testEnv[kv[0]] = kv.slice(1).join('='); + throw Error('Unknown test type: ' + test.type); + } + + let benchmarkResult = null; + if(options.benchmarkOptions) { + const result = await runBenchmark({ + test, + testInfo, + run, + params: testInfo.params.map(param => param(test, { + // pre-load params to avoid doc loader and parser timing + load: true + })), + mochaTest: self + }); + benchmarkResult = { + // FIXME use generic prefix + '@type': 'jldb:BenchmarkResult', + 'jldb:hz': result.target.hz, + 'jldb:rme': result.target.stats.rme + }; + } + + if(options.earl.report) { + options.earl.report.addAssertion(test, true, { + benchmarkResult + }); + } + } catch(err) { + // FIXME: improve handling of non-normative errors + // FIXME: for now, explicitly disabling tests. + //if(!normativeTest) { + // // failure ok + // if(options.verboseSkip) { + // console.log('Skipping non-normative test due to failure:', + // {id: test['@id'], name: test.name}); + // } + // self.skip(); + //} + if(options.bailOnError) { + if(err.name !== 'AssertionError') { + console.error('\nError: ', JSON.stringify(err, null, 2)); } + options.exit(); + } + if(options.earl.report) { + options.earl.report.addAssertion(test, false); + } + console.error('Error: ', JSON.stringify(err, null, 2)); + throw err; + } + }; +} + +async function runBenchmark({test, testInfo, params, run, mochaTest}) { + const values = await Promise.all(params); + + return new Promise((resolve, reject) => { + const suite = new benchmark.Suite(); + suite.add({ + name: test.name, + defer: true, + fn: deferred => { + run({test, testInfo, params: values}).then(() => { + deferred.resolve(); + }); } }); - if(testEnv.label === 'auto') { - testEnv.label = ''; + suite + .on('start', e => { + // set timeout to a bit more than max benchmark time + mochaTest.timeout((e.target.maxTime + 2) * 1000); + }) + .on('cycle', e => { + console.log(String(e.target)); + }) + .on('error', err => { + reject(new Error(err)); + }) + .on('complete', e => { + resolve(e); + }) + .run({async: true}); + }); +} + +function getJsonLdTestType(test) { + const types = Object.keys(TEST_TYPES); + for(let i = 0; i < types.length; ++i) { + if(isJsonLdType(test, types[i])) { + return types[i]; } - if(testEnv.arch === 'auto') { - testEnv.arch = process.arch; + } + return null; +} + +function readManifestEntry(manifest, entry) { + let p = Promise.resolve(); + let _entry = entry; + if(typeof entry === 'string') { + let _filename; + p = p.then(() => { + if(entry.endsWith('json') || entry.endsWith('jsonld')) { + // load as file + return entry; + } + // load as dir with manifest.jsonld + return joinPath(entry, 'manifest.jsonld'); + }).then(entry => { + const dir = dirname(manifest.filename); + return joinPath(dir, entry); + }).then(filename => { + _filename = filename; + return readJson(filename); + }).then(entry => { + _entry = entry; + _entry.filename = _filename; + return _entry; + }).catch(err => { + if(err.code === 'ENOENT') { + //console.log('File does not exist, skipping: ' + _filename); + // return a "skip" entry + _entry = { + type: '__SKIP__', + title: 'Not found, skipping: ' + _filename, + filename: _filename, + skip: true + }; + return; + } + throw err; + }); + } + return p.then(() => { + _entry.dirname = dirname(_entry.filename || manifest.filename); + return _entry; + }); +} + +function readTestUrl(property) { + return async function(test, options) { + if(!test[property]) { + return null; } - if(testEnv.cpu === 'auto') { - testEnv.cpu = os.cpus()[0].model; + if(options && options.load) { + // always load + const filename = await joinPath(test.dirname, test[property]); + return readJson(filename); } - if(testEnv.cpuCount === 'auto') { - testEnv.cpuCount = os.cpus().length; + return test.manifest.baseIri + test[property]; + }; +} + +function readTestJson(property) { + return async function(test) { + if(!test[property]) { + return null; } - if(testEnv.platform === 'auto') { - testEnv.platform = process.platform; + const filename = await joinPath(test.dirname, test[property]); + return readJson(filename); + }; +} + +function readTestNQuads(property) { + return async function(test) { + if(!test[property]) { + return null; + } + const filename = await joinPath(test.dirname, test[property]); + return readFile(filename); + }; +} + +function createTestOptions(opts) { + return function(test) { + const options = { + documentLoader: createDocumentLoader(test) + }; + const httpOptions = ['contentType', 'httpLink', 'httpStatus', 'redirectTo']; + const testOptions = test.option || {}; + for(const key in testOptions) { + if(httpOptions.indexOf(key) === -1) { + options[key] = testOptions[key]; + } } - if(testEnv.runtime === 'auto') { - testEnv.runtime = 'Node.js'; + if(opts) { + // extend options + for(const key in opts) { + options[key] = opts[key]; + } } - if(testEnv.runtimeVersion === 'auto') { - testEnv.runtimeVersion = process.version; + return options; + }; +} + +// find the expected output property or throw error +function _getExpectProperty(test) { + if('expectErrorCode' in test) { + return 'expectErrorCode'; + } else if('expect' in test) { + return 'expect'; + } else if('result' in test) { + return 'result'; + } else { + throw Error('No expected output property found'); + } +} + +async function compareExpectedJson(test, result) { + let expect; + try { + expect = await readTestJson(_getExpectProperty(test))(test); + assert.deepStrictEqual(result, expect); + } catch(err) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED: ' + JSON.stringify(expect, null, 2)); + console.log('ACTUAL: ' + JSON.stringify(result, null, 2)); } - if(testEnv.comment === 'auto') { - testEnv.comment = ''; + throw err; + } +} + +async function compareExpectedNQuads(test, result) { + let expect; + try { + expect = await readTestNQuads(_getExpectProperty(test))(test); + assert.strictEqual(result, expect); + } catch(ex) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED:\n' + expect); + console.log('ACTUAL:\n' + result); } - if(testEnv.version === 'auto') { - testEnv.version = require('../package.json').version; + throw ex; + } +} + +async function compareCanonizedExpectedNQuads(test, result) { + let expect; + try { + expect = await readTestNQuads(_getExpectProperty(test))(test); + const opts = {algorithm: 'URDNA2015'}; + const expectDataset = rdfCanonize.NQuads.parse(expect); + const expectCmp = await rdfCanonize.canonize(expectDataset, opts); + const resultDataset = rdfCanonize.NQuads.parse(result); + const resultCmp = await rdfCanonize.canonize(resultDataset, opts); + assert.strictEqual(resultCmp, expectCmp); + } catch(err) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED:\n' + expect); + console.log('ACTUAL:\n' + result); } + throw err; } } -let benchmarkOptions = null; -if(process.env.JSONLD_BENCHMARK) { - if(!(['0', 'false'].includes(process.env.JSONLD_BENCHMARK))) { - benchmarkOptions = {}; - if(!(['1', 'true'].includes(process.env.JSONLD_BENCHMARK))) { - process.env.JSONLD_BENCHMARK.split(',').forEach(pair => { - const kv = pair.split('='); - benchmarkOptions[kv[0]] = kv[1]; - }); +async function compareExpectedError(test, err) { + let expect; + let result; + try { + expect = test[_getExpectProperty(test)]; + result = getJsonLdErrorCode(err); + assert.ok(err, 'no error present'); + assert.strictEqual(result, expect); + } catch(_err) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED: ' + expect); + console.log('ACTUAL: ' + result); } + // log the unexpected error to help with debugging + console.log('Unexpected error:', err); + throw _err; } } -const options = { - nodejs: { - path - }, - assert, - benchmark, - jsonld, - exit: code => process.exit(code), - earl: { - filename: process.env.EARL - }, - verboseSkip: process.env.VERBOSE_SKIP === 'true', - bailOnError: process.env.BAIL === 'true', - entries, - testEnv, - benchmarkOptions, - readFile: filename => { - return fs.readFile(filename, 'utf8'); - }, - writeFile: (filename, data) => { - return fs.outputFile(filename, data); - }, - import: f => require(f) -}; +function isJsonLdType(node, type) { + const nodeType = [].concat( + getJsonLdValues(node, '@type'), + getJsonLdValues(node, 'type') + ); + type = Array.isArray(type) ? type : [type]; + for(let i = 0; i < type.length; ++i) { + if(nodeType.indexOf(type[i]) !== -1) { + return true; + } + } + return false; +} -// wait for setup of all tests then run mocha -common(options).then(() => { - run(); -}).catch(err => { - console.error(err); -}); +function getJsonLdValues(node, property) { + let rval = []; + if(property in node) { + rval = node[property]; + if(!Array.isArray(rval)) { + rval = [rval]; + } + } + return rval; +} -process.on('unhandledRejection', (reason, p) => { - console.error('Unhandled Rejection at:', p, 'reason:', reason); -}); +function getJsonLdErrorCode(err) { + if(!err) { + return null; + } + if(err.details) { + if(err.details.code) { + return err.details.code; + } + if(err.details.cause) { + return getJsonLdErrorCode(err.details.cause); + } + } + return err.name; +} + +async function readJson(filename) { + const data = await readFile(filename); + return JSON.parse(data); +} + +async function readFile(filename) { + return options.readFile(filename); +} + +async function joinPath() { + return join.apply(null, Array.prototype.slice.call(arguments)); +} + +function dirname(filename) { + if(options.nodejs) { + return options.nodejs.path.dirname(filename); + } + const idx = filename.lastIndexOf('/'); + if(idx === -1) { + return filename; + } + return filename.substr(0, idx); +} + +function basename(filename) { + if(options.nodejs) { + return options.nodejs.path.basename(filename); + } + const idx = filename.lastIndexOf('/'); + if(idx === -1) { + return filename; + } + return filename.substr(idx + 1); +} + +// check test.option.loader.rewrite map for url, +// if no test rewrite, check manifest, +// else no rewrite +function rewrite(test, url) { + if(test.option && + test.option.loader && + test.option.loader.rewrite && + url in test.option.loader.rewrite) { + return test.option.loader.rewrite[url]; + } + const manifest = test.manifest; + if(manifest.option && + manifest.option.loader && + manifest.option.loader.rewrite && + url in manifest.option.loader.rewrite) { + return manifest.option.loader.rewrite[url]; + } + return url; +} + +/** + * Creates a test remote document loader. + * + * @param test the test to use the document loader for. + * + * @return the document loader. + */ +function createDocumentLoader(test) { + const localBases = [ + 'http://json-ld.org/test-suite', + 'https://json-ld.org/test-suite', + 'https://json-ld.org/benchmarks', + 'https://w3c.github.io/json-ld-api/tests', + 'https://w3c.github.io/json-ld-framing/tests' + ]; + + const localLoader = function(url) { + // always load remote-doc tests remotely in node + // NOTE: disabled due to github pages issues. + //if(options.nodejs && test.manifest.name === 'Remote document') { + // return jsonld.documentLoader(url); + //} + + // handle loader rewrite options for test or manifest + url = rewrite(test, url); + + // FIXME: this check only works for main test suite and will not work if: + // - running other tests and main test suite not installed + // - use other absolute URIs but want to load local files + const isTestSuite = localBases.some(function(base) { + return url.startsWith(base); + }); + // TODO: improve this check + const isRelative = url.indexOf(':') === -1; + if(isTestSuite || isRelative) { + // attempt to load official test-suite files or relative URLs locally + return loadLocally(url); + } + + // load remotely + return jsonld.documentLoader(url); + }; + + return localLoader; + + function loadLocally(url) { + const doc = {contextUrl: null, documentUrl: url, document: null}; + const options = test.option; + if(options && url === test.base) { + if('redirectTo' in options && parseInt(options.httpStatus, 10) >= 300) { + doc.documentUrl = test.manifest.baseIri + options.redirectTo; + } else if('httpLink' in options) { + let contentType = options.contentType || null; + if(!contentType && url.indexOf('.jsonld', url.length - 7) !== -1) { + contentType = 'application/ld+json'; + } + if(!contentType && url.indexOf('.json', url.length - 5) !== -1) { + contentType = 'application/json'; + } + let linkHeader = options.httpLink; + if(Array.isArray(linkHeader)) { + linkHeader = linkHeader.join(','); + } + const linkHeaders = jsonld.parseLinkHeader(linkHeader); + const linkedContext = + linkHeaders['http://www.w3.org/ns/json-ld#context']; + if(linkedContext && contentType !== 'application/ld+json') { + if(Array.isArray(linkedContext)) { + throw {name: 'multiple context link headers'}; + } + doc.contextUrl = linkedContext.target; + } + + // If not JSON-LD, alternate may point there + if(linkHeaders.alternate && + linkHeaders.alternate.type == 'application/ld+json' && + !(contentType || '').match(/^application\/(\w*\+)?json$/)) { + doc.documentUrl = prependBase(url, linkHeaders.alternate.target); + } + } + } + + let p = Promise.resolve(); + if(doc.documentUrl.indexOf(':') === -1) { + p = p.then(() => { + return joinPath(test.manifest.dirname, doc.documentUrl); + }).then(filename => { + doc.documentUrl = 'file://' + filename; + return filename; + }); + } else { + p = p.then(() => { + return joinPath( + test.manifest.dirname, + doc.documentUrl.substr(test.manifest.baseIri.length)); + }).then(fn => { + return fn; + }); + } + + return p.then(readJson).then(json => { + doc.document = json; + return doc; + }).catch(() => { + throw {name: 'loading document failed', url}; + }); + } +} + +};