forked from finos/legend
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
remove website/.docusaurus and clean up scripts (finos#623)
- Loading branch information
Showing
64 changed files
with
127 additions
and
7,269 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
179 changes: 118 additions & 61 deletions
179
scripts/check-legend-application-documentation-broken-links.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,81 +1,138 @@ | ||
| const fs = require("fs"); | ||
| const http = require("http"); | ||
| const https = require("https"); | ||
| const path = require("path"); | ||
| const { parse } = require("url"); | ||
|
|
||
| const DOC_WEBSITE_URL = "https://legend.finos.org/"; | ||
| const APPLICATION_DOC_DIRECTORY = | ||
| "website/static/resource/studio/documentation/"; | ||
| const WEBSITE_CONTENT_DIRECTORY = "./website/build/"; | ||
| const APPLICATION_DOC_DIRECTORY = path.resolve( | ||
| __dirname, | ||
| "../website/static/resource/studio/documentation/" | ||
| ); | ||
| const WEBSITE_CONTENT_DIRECTORY = path.resolve(__dirname, "../website/build/"); | ||
|
|
||
| var files = fs.readdirSync(APPLICATION_DOC_DIRECTORY); | ||
| var brokenUrls = []; | ||
| async function detectBrokenLinks() { | ||
| const files = fs.readdirSync(APPLICATION_DOC_DIRECTORY); | ||
|
|
||
| for (let i = 0; i < files.length; i++) { | ||
| if (!files[i].endsWith(".json")) continue; | ||
| var jsonObject = JSON.parse( | ||
| fs.readFileSync(APPLICATION_DOC_DIRECTORY + files[i], "utf8") | ||
| ); | ||
| const brokenLinks = ( | ||
| await Promise.all( | ||
| files.flatMap((file) => { | ||
| const fileContent = JSON.parse( | ||
| fs.readFileSync(path.resolve(APPLICATION_DOC_DIRECTORY, file), "utf8") | ||
| ); | ||
|
|
||
| const keys = Object.keys(jsonObject.entries); | ||
| for (let i = 0; i < keys.length; i++) { | ||
| const key = keys[i]; | ||
| if (jsonObject.entries[key].url == undefined) { | ||
| continue; | ||
| } | ||
| lookForUrl(jsonObject.entries[key].url); | ||
| } | ||
| } | ||
| return Object.values(fileContent.entries) | ||
| .filter((entry) => entry.url) | ||
| .map((entry) => checkLink(entry.url)); | ||
| }) | ||
| ) | ||
| ).filter(Boolean); | ||
|
|
||
| if (brokenUrls.length !== 0) { | ||
| throw new Error("Broken link(s) found: " + brokenUrls); | ||
| if (brokenLinks.length !== 0) { | ||
| throw new Error( | ||
| `Found ${brokenLinks.length} broken link(s):\n${brokenLinks.join("\n")}` | ||
| ); | ||
| } else { | ||
| console.log(`No broken links found!`); | ||
| } | ||
| } | ||
|
|
||
| async function lookForUrlWithAnchor(url) { | ||
| const locationOfFile = url | ||
| .substring(0, url.lastIndexOf("/")) | ||
| .replace(DOC_WEBSITE_URL, ""); | ||
|
|
||
| const filePathName = | ||
| WEBSITE_CONTENT_DIRECTORY + | ||
| locationOfFile + | ||
| url.substring(url.lastIndexOf("/"), url.lastIndexOf("#")) + | ||
| ".html"; | ||
|
|
||
| const anchorTag = url.substring(url.lastIndexOf("#")); | ||
|
|
||
| if (await fs.existsSync(filePathName)) { | ||
| await fs.readFile(filePathName, "UTF-8", (err, data) => { | ||
| if (!data) { | ||
| collectionOfErrors.push(["\n" + filePathName + "\n"]); | ||
| // Use `HEAD` request via http to check if the url still exists | ||
| // See https://stackoverflow.com/questions/60841965/how-can-i-check-to-see-if-a-url-exists-or-not | ||
| // See https://github.com/nwaughachukwuma/url-exists-nodejs | ||
| async function checkExternalLinkExists(url) { | ||
| return new Promise((resolve) => { | ||
| const req = http.request( | ||
| { | ||
| method: "HEAD", | ||
| host: parse(url).host, | ||
| path: parse(url).pathname, | ||
| port: 80, | ||
| }, | ||
| (response) => { | ||
| // NOTE: only 400 requests are considered not found | ||
| resolve(response.statusCode < 400 || response.statusCode >= 500); | ||
| } | ||
| ); | ||
|
|
||
| if (!data.toLocaleLowerCase().includes(anchorTag)) { | ||
| brokenUrls.push(["\n" + url + "\n"]); | ||
| throw new Error("Broken link(s) found: " + brokenUrls); | ||
| } | ||
| }); | ||
| } else { | ||
| brokenUrls.push(["\n" + url + "\n"]); | ||
| } | ||
| req.end(); | ||
| }); | ||
| } | ||
|
|
||
| async function lookForUrl(url) { | ||
| currUrl = url.replace(DOC_WEBSITE_URL, ""); | ||
| lastDirectoryIndex = currUrl.lastIndexOf("/"); | ||
| async function fetchExternalLinkSiteData(url) { | ||
| return new Promise((resolve, reject) => { | ||
| https.get(url, (response) => { | ||
| let chunks_of_data = []; | ||
| response.on("data", (fragments) => { | ||
| chunks_of_data.push(fragments); | ||
| }); | ||
| response.on("end", () => { | ||
| let response_body = Buffer.concat(chunks_of_data); | ||
| resolve(response_body.toString()); | ||
| }); | ||
| response.on("error", (error) => { | ||
| reject(error); | ||
| }); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| fileName = currUrl.substring(lastDirectoryIndex); | ||
| currUrlFinal = currUrl.substring(0, lastDirectoryIndex); | ||
| async function checkLink(url) { | ||
| if (!url.startsWith(DOC_WEBSITE_URL)) { | ||
| // we will not handle unsecured links | ||
| if (!url.startsWith("https://")) { | ||
| return undefined; | ||
| } | ||
|
|
||
| if (fs.existsSync(WEBSITE_CONTENT_DIRECTORY + currUrlFinal)) { | ||
| if (fileName.lastIndexOf("#") !== -1) { | ||
| //includes markdown tag | ||
| lookForUrlWithAnchor(url); | ||
| let result = undefined; | ||
| const urlExists = await checkExternalLinkExists(url); | ||
| if (urlExists) { | ||
| await fetchExternalLinkSiteData(url) | ||
| .then((data) => { | ||
| if ( | ||
| url.lastIndexOf("#") !== -1 && | ||
| !data.includes(url.substring(url.lastIndexOf("#"))) | ||
| ) { | ||
| result = url; | ||
| } | ||
| }) | ||
| .catch(() => { | ||
| result = url; | ||
| }); | ||
| } else { | ||
| if (!fs.existsSync(currUrlFinal + fileName + ".md")) { | ||
| brokenUrls.push(["\n" + url + "\n"]); | ||
| } | ||
| result = url; | ||
| } | ||
| } else { | ||
| brokenUrls.push(["\n" + url + "\n"]); | ||
| return result; | ||
| } | ||
|
|
||
| return; | ||
| const relativeUrl = url.substring(DOC_WEBSITE_URL.length); | ||
| const anchorIdx = relativeUrl.lastIndexOf("#"); | ||
| const anchorTag = | ||
| anchorIdx !== -1 ? relativeUrl.substring(anchorIdx) : undefined; | ||
| const filePath = | ||
| anchorIdx !== -1 | ||
| ? path.resolve( | ||
| WEBSITE_CONTENT_DIRECTORY, | ||
| `${relativeUrl.substring(0, anchorIdx)}.html` | ||
| ) | ||
| : path.resolve(WEBSITE_CONTENT_DIRECTORY, `${relativeUrl}.html`); | ||
|
|
||
| if (fs.existsSync(filePath)) { | ||
| if (anchorTag) { | ||
| try { | ||
| const data = fs.readFileSync(filePath, { | ||
| encoding: "utf-8", | ||
| }); | ||
| if (data && !data.toLocaleLowerCase().includes(anchorTag)) { | ||
| return url; | ||
| } | ||
| } catch { | ||
| return url; | ||
| } | ||
| } | ||
| return undefined; | ||
| } | ||
| return url; | ||
| } | ||
|
|
||
| detectBrokenLinks(); |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.