From 7ac03daa3a5da01f83965f205b56ac03ae55fa36 Mon Sep 17 00:00:00 2001 From: Sam Ruby Date: Tue, 3 Jul 2018 12:46:56 -0400 Subject: [PATCH] tools: build all.json by combining generated JSON Notes: 1) Removed a number of root properties that did not seem relevant: source, desc, and introduced_in. There no longer is a source, and the other two are from the first include and do not reflect the entire API. 2) As with https://github.com/nodejs/node/issues/20100, the current "desc" properties sometimes contained in-page links, other times referenced another page, and often did not match the links in the original HTML or JSON file. I chose to standardize on external links as "desc" values are isolated snippets as opposed to all.html which can be viewed as a standalone and self contained document. 3) Eliminated preprocessing for @include entirely, including the test case for this function. 4) _toc.md was renamed to index.md. 5) index comments no longer appear in embedded TOCs (left hand side column in the generated documentation. --- Makefile | 8 ++-- doc/api/_toc.md | 62 ---------------------------- doc/api/all.md | 50 ----------------------- doc/api/index.md | 65 +++++++++++++++++++++++++++++- test/doctool/test-doctool-html.js | 64 ++++++++++++----------------- test/doctool/test-make-doc.js | 10 ++--- test/fixtures/doc_with_includes.md | 2 - tools/doc/allhtml.js | 13 +++--- tools/doc/alljson.js | 56 +++++++++++++++++++++++++ tools/doc/generate.js | 9 +---- tools/doc/html.js | 4 +- tools/doc/preprocess.js | 38 ----------------- 12 files changed, 165 insertions(+), 216 deletions(-) delete mode 100644 doc/api/_toc.md delete mode 100644 doc/api/all.md delete mode 100644 test/fixtures/doc_with_includes.md create mode 100644 tools/doc/alljson.js delete mode 100644 tools/doc/preprocess.js diff --git a/Makefile b/Makefile index 634eec5151dc44..600d87fd2316b2 100644 --- a/Makefile +++ b/Makefile @@ -615,7 +615,7 @@ doc-only: $(apidoc_dirs) $(apiassets) ## Builds the docs with the local or the if [ ! -d doc/api/assets ]; then \ $(MAKE) tools/doc/node_modules/js-yaml/package.json; \ fi; - @$(MAKE) $(apidocs_html) $(apidocs_json) + @$(MAKE) out/doc/api/all.html out/doc/api/all.json .PHONY: doc doc: $(NODE_EXE) doc-only @@ -665,10 +665,12 @@ out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js $(call available-node, $(gen-html)) -out/doc/api/all.html: $(filter-out out/doc/api/all.html, $(apidocs_html)) \ - tools/doc/allhtml.js +out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js $(call available-node, tools/doc/allhtml.js) +out/doc/api/all.json: $(apidocs_json) tools/doc/alljson.js + $(call available-node, tools/doc/alljson.js) + .PHONY: docopen docopen: $(apidocs_html) @$(PYTHON) -mwebbrowser file://$(PWD)/out/doc/api/all.html diff --git a/doc/api/_toc.md b/doc/api/_toc.md deleted file mode 100644 index e307d52ae8544e..00000000000000 --- a/doc/api/_toc.md +++ /dev/null @@ -1,62 +0,0 @@ -@// NB(chrisdickinson): if you move this file, be sure to update -@// tools/doc/html.js to point at the new location. - - - -* [About these Docs](documentation.html) -* [Usage & Example](synopsis.html) - -
- -* [Assertion Testing](assert.html) -* [Async Hooks](async_hooks.html) -* [Buffer](buffer.html) -* [C++ Addons](addons.html) -* [C/C++ Addons - N-API](n-api.html) -* [Child Processes](child_process.html) -* [Cluster](cluster.html) -* [Command Line Options](cli.html) -* [Console](console.html) -* [Crypto](crypto.html) -* [Debugger](debugger.html) -* [Deprecated APIs](deprecations.html) -* [DNS](dns.html) -* [Domain](domain.html) -* [ECMAScript Modules](esm.html) -* [Errors](errors.html) -* [Events](events.html) -* [File System](fs.html) -* [Globals](globals.html) -* [HTTP](http.html) -* [HTTP/2](http2.html) -* [HTTPS](https.html) -* [Inspector](inspector.html) -* [Internationalization](intl.html) -* [Modules](modules.html) -* [Net](net.html) -* [OS](os.html) -* [Path](path.html) -* [Performance Hooks](perf_hooks.html) -* [Process](process.html) -* [Punycode](punycode.html) -* [Query Strings](querystring.html) -* [Readline](readline.html) -* [REPL](repl.html) -* [Stream](stream.html) -* [String Decoder](string_decoder.html) -* [Timers](timers.html) -* [TLS/SSL](tls.html) -* [Trace Events](tracing.html) -* [TTY](tty.html) -* [UDP/Datagram](dgram.html) -* [URL](url.html) -* [Utilities](util.html) -* [V8](v8.html) -* [VM](vm.html) -* [Worker Threads](worker_threads.html) -* [ZLIB](zlib.html) - -
- -* [GitHub Repo & Issue Tracker](https://github.com/nodejs/node) -* [Mailing List](https://groups.google.com/group/nodejs) diff --git a/doc/api/all.md b/doc/api/all.md deleted file mode 100644 index 47216b695d3351..00000000000000 --- a/doc/api/all.md +++ /dev/null @@ -1,50 +0,0 @@ - -@include documentation -@include synopsis -@include assert -@include async_hooks -@include buffer -@include addons -@include n-api -@include child_process -@include cluster -@include cli -@include console -@include crypto -@include debugger -@include deprecations -@include dns -@include domain -@include esm -@include errors -@include events -@include fs -@include globals -@include http -@include http2 -@include https -@include inspector -@include intl -@include modules -@include net -@include os -@include path -@include perf_hooks -@include process -@include punycode -@include querystring -@include readline -@include repl -@include stream -@include string_decoder -@include timers -@include tls -@include tracing -@include tty -@include dgram -@include url -@include util -@include v8 -@include vm -@include worker_threads -@include zlib diff --git a/doc/api/index.md b/doc/api/index.md index 400faf6e5ed34a..0b43097bf294de 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -1 +1,64 @@ -@include _toc.md + + + + +* [About these Docs](documentation.html) +* [Usage & Example](synopsis.html) + +
+ +* [Assertion Testing](assert.html) +* [Async Hooks](async_hooks.html) +* [Buffer](buffer.html) +* [C++ Addons](addons.html) +* [C/C++ Addons - N-API](n-api.html) +* [Child Processes](child_process.html) +* [Cluster](cluster.html) +* [Command Line Options](cli.html) +* [Console](console.html) +* [Crypto](crypto.html) +* [Debugger](debugger.html) +* [Deprecated APIs](deprecations.html) +* [DNS](dns.html) +* [Domain](domain.html) +* [ECMAScript Modules](esm.html) +* [Errors](errors.html) +* [Events](events.html) +* [File System](fs.html) +* [Globals](globals.html) +* [HTTP](http.html) +* [HTTP/2](http2.html) +* [HTTPS](https.html) +* [Inspector](inspector.html) +* [Internationalization](intl.html) +* [Modules](modules.html) +* [Net](net.html) +* [OS](os.html) +* [Path](path.html) +* [Performance Hooks](perf_hooks.html) +* [Process](process.html) +* [Punycode](punycode.html) +* [Query Strings](querystring.html) +* [Readline](readline.html) +* [REPL](repl.html) +* [Stream](stream.html) +* [String Decoder](string_decoder.html) +* [Timers](timers.html) +* [TLS/SSL](tls.html) +* [Trace Events](tracing.html) +* [TTY](tty.html) +* [UDP/Datagram](dgram.html) +* [URL](url.html) +* [Utilities](util.html) +* [V8](v8.html) +* [VM](vm.html) +* [Worker Threads](worker_threads.html) +* [ZLIB](zlib.html) + +
+ +* [GitHub Repo & Issue Tracker](https://github.com/nodejs/node) +* [Mailing List](https://groups.google.com/group/nodejs) diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js index 0ca46c0adedfc1..a6119c16903a74 100644 --- a/test/doctool/test-doctool-html.js +++ b/test/doctool/test-doctool-html.js @@ -11,7 +11,6 @@ try { const assert = require('assert'); const { readFile } = require('fs'); const fixtures = require('../common/fixtures'); -const processIncludes = require('../../tools/doc/preprocess.js'); const toHTML = require('../../tools/doc/html.js'); // Test data is a list of objects with two properties. @@ -64,15 +63,6 @@ const testData = [ ' ' + '

Describe Something in more detail here.

' }, - { - file: fixtures.path('doc_with_includes.md'), - html: '' + - '

Look here!

' + - '' + - '

foobar#

' + - '

I exist and am being linked to.

' - }, { file: fixtures.path('sample_document.md'), html: '
  1. fish

  2. fish

  3. Redfish

    4. ' + @@ -90,36 +80,34 @@ testData.forEach(({ file, html, analyticsId }) => { readFile(file, 'utf8', common.mustCall((err, input) => { assert.ifError(err); - processIncludes(file, input, common.mustCall((err, preprocessed) => { - assert.ifError(err); - toHTML( - { - input: preprocessed, - filename: 'foo', - nodeVersion: process.version, - analytics: analyticsId, - }, - common.mustCall((err, output) => { - assert.ifError(err); + toHTML( + { + input: input, + filename: 'foo', + nodeVersion: process.version, + analytics: analyticsId, + }, + common.mustCall((err, output) => { + assert.ifError(err); - const actual = output.replace(spaces, ''); - // Assert that the input stripped of all whitespace contains the - // expected markup. - assert(actual.includes(expected)); + const actual = output.replace(spaces, ''); + // Assert that the input stripped of all whitespace contains the + // expected markup. + assert(actual.includes(expected)); - // Testing the insertion of Google Analytics script when - // an analytics id is provided. Should not be present by default. - const scriptDomain = 'google-analytics.com'; - if (includeAnalytics) { - assert(actual.includes(scriptDomain), - `Google Analytics script was not present in "${actual}"`); - } else { - assert.strictEqual(actual.includes(scriptDomain), false, - 'Google Analytics script was present in ' + - `"${actual}"`); - } - })); - })); + // Testing the insertion of Google Analytics script when + // an analytics id is provided. Should not be present by default. + const scriptDomain = 'google-analytics.com'; + if (includeAnalytics) { + assert(actual.includes(scriptDomain), + `Google Analytics script was not present in "${actual}"`); + } else { + assert.strictEqual(actual.includes(scriptDomain), false, + 'Google Analytics script was present in ' + + `"${actual}"`); + } + }) + ); })); }); diff --git a/test/doctool/test-make-doc.js b/test/doctool/test-make-doc.js index e019288f75cc1e..a11da0e97d8355 100644 --- a/test/doctool/test-make-doc.js +++ b/test/doctool/test-make-doc.js @@ -13,17 +13,16 @@ const path = require('path'); const apiPath = path.resolve(__dirname, '..', '..', 'out', 'doc', 'api'); const allDocs = fs.readdirSync(apiPath); -assert.ok(allDocs.includes('_toc.html')); +assert.ok(allDocs.includes('index.html')); const actualDocs = allDocs.filter( (name) => { const extension = path.extname(name); - return (extension === '.html' || extension === '.json') && - name !== '_toc.html'; + return extension === '.html' || extension === '.json'; } ); -const toc = fs.readFileSync(path.resolve(apiPath, '_toc.html'), 'utf8'); +const toc = fs.readFileSync(path.resolve(apiPath, 'index.html'), 'utf8'); const re = /href="([^/]+\.html)"/; const globalRe = new RegExp(re, 'g'); const links = toc.match(globalRe); @@ -32,8 +31,7 @@ assert.notStrictEqual(links, null); // Filter out duplicate links, leave just filenames, add expected JSON files. const linkedHtmls = [...new Set(links)].map((link) => link.match(re)[1]); const expectedJsons = linkedHtmls - .map((name) => name.replace('.html', '.json')) - .concat('_toc.json'); + .map((name) => name.replace('.html', '.json')); const expectedDocs = linkedHtmls.concat(expectedJsons); // Test that all the relative links in the TOC match to the actual documents. diff --git a/test/fixtures/doc_with_includes.md b/test/fixtures/doc_with_includes.md deleted file mode 100644 index 901bf0f1b0bf3b..00000000000000 --- a/test/fixtures/doc_with_includes.md +++ /dev/null @@ -1,2 +0,0 @@ -@include doc_inc_1 -@include doc_inc_2.md diff --git a/tools/doc/allhtml.js b/tools/doc/allhtml.js index fad530f1c44245..1c84e13d0ab79c 100644 --- a/tools/doc/allhtml.js +++ b/tools/doc/allhtml.js @@ -12,7 +12,7 @@ const htmlFiles = fs.readdirSync(source, 'utf8') .filter((name) => name.includes('.html') && name !== 'all.html'); // Read the table of contents. -const toc = fs.readFileSync(source + '/_toc.html', 'utf8'); +const toc = fs.readFileSync(source + '/index.html', 'utf8'); // Extract (and concatenate) the toc and apicontent from each document. let contents = ''; @@ -47,11 +47,12 @@ for (const link of toc.match(//g)) { seen[href] = true; } -// Replace various mentions of _toc with all. -let all = toc.replace(/_toc\.html/g, 'all.html') - .replace('_toc.json', 'all.json') - .replace('api-section-_toc', 'api-section-all') - .replace('data-id="_toc"', 'data-id="all"'); +// Replace various mentions of index with all. +let all = toc.replace(/index\.html/g, 'all.html') + .replace('', '') + .replace('index.json', 'all.json') + .replace('api-section-index', 'api-section-all') + .replace('data-id="index"', 'data-id="all"'); // Clean up the title. all = all.replace(/.*?\| /, '<title>'); diff --git a/tools/doc/alljson.js b/tools/doc/alljson.js new file mode 100644 index 00000000000000..7e027f764e7efd --- /dev/null +++ b/tools/doc/alljson.js @@ -0,0 +1,56 @@ +'use strict'; + +// Build all.json by combining the miscs, modules, classes, globals, and methods +// from the generated json files. + +const fs = require('fs'); + +const source = `${__dirname}/../../out/doc/api`; + +// Get a list of generated API documents. +const jsonFiles = fs.readdirSync(source, 'utf8') + .filter((name) => name.includes('.json') && name !== 'all.json'); + +// Read the table of contents. +const toc = fs.readFileSync(source + '/index.html', 'utf8'); + +// Initialize results. Only these four data values will be collected. +const results = { + miscs: [], + modules: [], + classes: [], + globals: [], + methods: [] +}; + +// Identify files that should be skipped. As files are processed, they +// are added to this list to prevent dupes. +const seen = { + 'all.json': true, + 'index.json': true +}; + +// Extract (and concatenate) the selected data from each document. +// Expand hrefs found in json to include source HTML file. +for (const link of toc.match(/<a.*?>/g)) { + const href = /href="(.*?)"/.exec(link)[1]; + const json = href.replace('.html', '.json'); + if (!jsonFiles.includes(json) || seen[json]) continue; + const data = JSON.parse( + fs.readFileSync(source + '/' + json, 'utf8') + .replace(/<a href=\\"#/g, `<a href=\\"${href}#`) + ); + + for (const property in data) { + if (results.hasOwnProperty(property)) { + results[property].push(...data[property]); + } + } + + // Mark source as seen. + seen[json] = true; +} + +// Write results. +fs.writeFileSync(source + '/all.json', + `${JSON.stringify(results, null, 2)}\n`, 'utf8'); diff --git a/tools/doc/generate.js b/tools/doc/generate.js index 9f217b19c7225f..8ed97610cf3bc2 100644 --- a/tools/doc/generate.js +++ b/tools/doc/generate.js @@ -21,7 +21,6 @@ 'use strict'; -const processIncludes = require('./preprocess.js'); const fs = require('fs'); // Parse the args. @@ -52,12 +51,6 @@ if (!filename) { } fs.readFile(filename, 'utf8', (er, input) => { - if (er) throw er; - // Process the input for @include lines. - processIncludes(filename, input, next); -}); - -function next(er, input) { if (er) throw er; switch (format) { case 'json': @@ -78,4 +71,4 @@ function next(er, input) { default: throw new Error(`Invalid format: ${format}`); } -} +}); diff --git a/tools/doc/html.js b/tools/doc/html.js index 871a55baf4676c..0e254f1203f7a6 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -36,8 +36,8 @@ marked.setOptions({ renderer }); const docPath = path.resolve(__dirname, '..', '..', 'doc'); -const gtocPath = path.join(docPath, 'api', '_toc.md'); -const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, ''); +const gtocPath = path.join(docPath, 'api', 'index.md'); +const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^<!--.*?-->/gms, ''); const gtocHTML = marked(gtocMD).replace( /<a href="(.*?)"/g, (all, href) => `<a class="nav-${href.replace('.html', '') diff --git a/tools/doc/preprocess.js b/tools/doc/preprocess.js deleted file mode 100644 index 554af2ccb77ae0..00000000000000 --- a/tools/doc/preprocess.js +++ /dev/null @@ -1,38 +0,0 @@ -'use strict'; - -module.exports = processIncludes; - -const path = require('path'); -const fs = require('fs'); - -const includeExpr = /^@include\s+([\w-]+)(?:\.md)?$/gmi; -const commentExpr = /^@\/\/.*$/gm; - -function processIncludes(inputFile, input, cb) { - const includes = input.match(includeExpr); - if (includes === null) - return cb(null, input.replace(commentExpr, '')); - - let errState = null; - let incCount = includes.length; - - includes.forEach((include) => { - const fname = include.replace(includeExpr, '$1.md'); - const fullFname = path.resolve(path.dirname(inputFile), fname); - - fs.readFile(fullFname, 'utf8', function(er, inc) { - if (errState) return; - if (er) return cb(errState = er); - incCount--; - - // Add comments to let the HTML generator know - // how the anchors for headings should look like. - inc = `<!-- [start-include:${fname}] -->\n` + - `${inc}\n<!-- [end-include:${fname}] -->\n`; - input = input.split(`${include}\n`).join(`${inc}\n`); - - if (incCount === 0) - return cb(null, input.replace(commentExpr, '')); - }); - }); -}