From efbe474e9c17f78ccbac1a5f01f288a304aa88f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Lorber?= Date: Thu, 28 Mar 2024 12:39:07 +0100 Subject: [PATCH] refactor(core): improve dev perf, fine-grained site reloads - part 3 (#9975) --- packages/docusaurus-types/src/context.d.ts | 1 + packages/docusaurus-types/src/plugin.d.ts | 3 +- packages/docusaurus-utils/src/emitUtils.ts | 10 +- packages/docusaurus-utils/src/gitUtils.ts | 15 +- packages/docusaurus/src/commands/build.ts | 208 ++++++++---------- .../docusaurus/src/commands/start/utils.ts | 18 +- .../src/commands/writeTranslations.ts | 4 +- .../__tests__/__snapshots__/site.test.ts.snap | 3 + .../server/__tests__/clientModules.test.ts | 16 +- .../src/server/__tests__/siteMetadata.test.ts | 23 +- .../docusaurus/src/server/clientModules.ts | 2 +- .../docusaurus/src/server/plugins/actions.ts | 1 + .../docusaurus/src/server/plugins/init.ts | 8 +- .../docusaurus/src/server/plugins/plugins.ts | 132 ++++++----- .../src/server/plugins/pluginsUtils.ts | 23 +- packages/docusaurus/src/server/site.ts | 56 ++--- .../docusaurus/src/server/siteMetadata.ts | 28 ++- .../__tests__/translations.test.ts | 14 +- .../src/server/translations/translations.ts | 27 ++- packages/docusaurus/src/ssg.ts | 27 +-- packages/docusaurus/src/utils.ts | 25 ++- project-words.txt | 1 + 22 files changed, 359 insertions(+), 286 deletions(-) diff --git a/packages/docusaurus-types/src/context.d.ts b/packages/docusaurus-types/src/context.d.ts index e05f8a9a3256..57481a98165a 100644 --- a/packages/docusaurus-types/src/context.d.ts +++ b/packages/docusaurus-types/src/context.d.ts @@ -31,6 +31,7 @@ export type GlobalData = {[pluginName: string]: {[pluginId: string]: unknown}}; export type LoadContext = { siteDir: string; + siteVersion: string | undefined; generatedFilesDir: string; siteConfig: DocusaurusConfig; siteConfigPath: string; diff --git a/packages/docusaurus-types/src/plugin.d.ts b/packages/docusaurus-types/src/plugin.d.ts index 44935f380788..fddc9c4fa5af 100644 --- a/packages/docusaurus-types/src/plugin.d.ts +++ b/packages/docusaurus-types/src/plugin.d.ts @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import type {TranslationFile} from './i18n'; +import type {CodeTranslations, TranslationFile} from './i18n'; import type {RuleSetRule, Configuration as WebpackConfiguration} from 'webpack'; import type {CustomizeRuleString} from 'webpack-merge/dist/types'; import type {CommanderStatic} from 'commander'; @@ -185,6 +185,7 @@ export type LoadedPlugin = InitializedPlugin & { readonly content: unknown; readonly globalData: unknown; readonly routes: RouteConfig[]; + readonly defaultCodeTranslations: CodeTranslations; }; export type PluginModule = { diff --git a/packages/docusaurus-utils/src/emitUtils.ts b/packages/docusaurus-utils/src/emitUtils.ts index ecce1f34861c..239569defb01 100644 --- a/packages/docusaurus-utils/src/emitUtils.ts +++ b/packages/docusaurus-utils/src/emitUtils.ts @@ -12,6 +12,10 @@ import {findAsyncSequential} from './jsUtils'; const fileHash = new Map(); +const hashContent = (content: string): string => { + return createHash('md5').update(content).digest('hex'); +}; + /** * Outputs a file to the generated files directory. Only writes files if content * differs from cache (for hot reload performance). @@ -38,7 +42,7 @@ export async function generate( // first "A" remains in cache. But if the file never existed in cache, no // need to register it. if (fileHash.get(filepath)) { - fileHash.set(filepath, createHash('md5').update(content).digest('hex')); + fileHash.set(filepath, hashContent(content)); } return; } @@ -50,11 +54,11 @@ export async function generate( // overwriting and we can reuse old file. if (!lastHash && (await fs.pathExists(filepath))) { const lastContent = await fs.readFile(filepath, 'utf8'); - lastHash = createHash('md5').update(lastContent).digest('hex'); + lastHash = hashContent(lastContent); fileHash.set(filepath, lastHash); } - const currentHash = createHash('md5').update(content).digest('hex'); + const currentHash = hashContent(content); if (lastHash !== currentHash) { await fs.outputFile(filepath, content); diff --git a/packages/docusaurus-utils/src/gitUtils.ts b/packages/docusaurus-utils/src/gitUtils.ts index fcc28df65009..869982b2241c 100644 --- a/packages/docusaurus-utils/src/gitUtils.ts +++ b/packages/docusaurus-utils/src/gitUtils.ts @@ -6,7 +6,16 @@ */ import path from 'path'; -import shell from 'shelljs'; +import fs from 'fs-extra'; +import _ from 'lodash'; +import shell from 'shelljs'; // TODO replace with async-first version + +const realHasGitFn = () => !!shell.which('git'); + +// The hasGit call is synchronous IO so we memoize it +// The user won't install Git in the middle of a build anyway... +const hasGit = + process.env.NODE_ENV === 'test' ? realHasGitFn : _.memoize(realHasGitFn); /** Custom error thrown when git is not found in `PATH`. */ export class GitNotFoundError extends Error {} @@ -86,13 +95,13 @@ export async function getFileCommitDate( timestamp: number; author?: string; }> { - if (!shell.which('git')) { + if (!hasGit()) { throw new GitNotFoundError( `Failed to retrieve git history for "${file}" because git is not installed.`, ); } - if (!shell.test('-f', file)) { + if (!(await fs.pathExists(file))) { throw new Error( `Failed to retrieve git history for "${file}" because the file does not exist.`, ); diff --git a/packages/docusaurus/src/commands/build.ts b/packages/docusaurus/src/commands/build.ts index cfd11c0c90a7..599f84de5fef 100644 --- a/packages/docusaurus/src/commands/build.ts +++ b/packages/docusaurus/src/commands/build.ts @@ -64,23 +64,15 @@ export async function build( process.on(sig, () => process.exit()); }); - async function tryToBuildLocale({ - locale, - isLastLocale, - }: { - locale: string; - isLastLocale: boolean; - }) { + async function tryToBuildLocale({locale}: {locale: string}) { try { - PerfLogger.start(`Building site for locale ${locale}`); - await buildLocale({ - siteDir, - locale, - cliOptions, - forceTerminate, - isLastLocale, - }); - PerfLogger.end(`Building site for locale ${locale}`); + await PerfLogger.async(`${logger.name(locale)}`, () => + buildLocale({ + siteDir, + locale, + cliOptions, + }), + ); } catch (err) { throw new Error( logger.interpolate`Unable to build website for locale name=${locale}.`, @@ -91,20 +83,28 @@ export async function build( } } - PerfLogger.start(`Get locales to build`); - const locales = await getLocalesToBuild({siteDir, cliOptions}); - PerfLogger.end(`Get locales to build`); + const locales = await PerfLogger.async('Get locales to build', () => + getLocalesToBuild({siteDir, cliOptions}), + ); if (locales.length > 1) { logger.info`Website will be built for all these locales: ${locales}`; } - PerfLogger.start(`Building ${locales.length} locales`); - await mapAsyncSequential(locales, (locale) => { - const isLastLocale = locales.indexOf(locale) === locales.length - 1; - return tryToBuildLocale({locale, isLastLocale}); - }); - PerfLogger.end(`Building ${locales.length} locales`); + await PerfLogger.async(`Build`, () => + mapAsyncSequential(locales, async (locale) => { + const isLastLocale = locales.indexOf(locale) === locales.length - 1; + await tryToBuildLocale({locale}); + if (isLastLocale) { + logger.info`Use code=${'npm run serve'} command to test your build locally.`; + } + + // TODO do we really need this historical forceTerminate exit??? + if (forceTerminate && isLastLocale && !cliOptions.bundleAnalyzer) { + process.exit(0); + } + }), + ); } async function getLocalesToBuild({ @@ -144,14 +144,10 @@ async function buildLocale({ siteDir, locale, cliOptions, - forceTerminate, - isLastLocale, }: { siteDir: string; locale: string; cliOptions: Partial; - forceTerminate: boolean; - isLastLocale: boolean; }): Promise { // Temporary workaround to unlock the ability to translate the site config // We'll remove it if a better official API can be designed @@ -160,81 +156,66 @@ async function buildLocale({ logger.info`name=${`[${locale}]`} Creating an optimized production build...`; - PerfLogger.start('Loading site'); - const site = await loadSite({ - siteDir, - outDir: cliOptions.outDir, - config: cliOptions.config, - locale, - localizePath: cliOptions.locale ? false : undefined, - }); - PerfLogger.end('Loading site'); + const site = await PerfLogger.async('Load site', () => + loadSite({ + siteDir, + outDir: cliOptions.outDir, + config: cliOptions.config, + locale, + localizePath: cliOptions.locale ? false : undefined, + }), + ); const {props} = site; const {outDir, plugins} = props; // We can build the 2 configs in parallel - PerfLogger.start('Creating webpack configs'); const [{clientConfig, clientManifestPath}, {serverConfig, serverBundlePath}] = - await Promise.all([ - getBuildClientConfig({ - props, - cliOptions, - }), - getBuildServerConfig({ - props, - }), - ]); - PerfLogger.end('Creating webpack configs'); - - // Make sure generated client-manifest is cleaned first, so we don't reuse - // the one from previous builds. - // TODO do we really need this? .docusaurus folder is cleaned between builds - PerfLogger.start('Deleting previous client manifest'); - await ensureUnlink(clientManifestPath); - PerfLogger.end('Deleting previous client manifest'); + await PerfLogger.async('Creating webpack configs', () => + Promise.all([ + getBuildClientConfig({ + props, + cliOptions, + }), + getBuildServerConfig({ + props, + }), + ]), + ); // Run webpack to build JS bundle (client) and static html files (server). - PerfLogger.start('Bundling'); - await compile([clientConfig, serverConfig]); - PerfLogger.end('Bundling'); + await PerfLogger.async('Bundling with Webpack', () => + compile([clientConfig, serverConfig]), + ); - PerfLogger.start('Executing static site generation'); - const {collectedData} = await executeSSG({ - props, - serverBundlePath, - clientManifestPath, - }); - PerfLogger.end('Executing static site generation'); + const {collectedData} = await PerfLogger.async('SSG', () => + executeSSG({ + props, + serverBundlePath, + clientManifestPath, + }), + ); // Remove server.bundle.js because it is not needed. - PerfLogger.start('Deleting server bundle'); - await ensureUnlink(serverBundlePath); - PerfLogger.end('Deleting server bundle'); + await PerfLogger.async('Deleting server bundle', () => + ensureUnlink(serverBundlePath), + ); // Plugin Lifecycle - postBuild. - PerfLogger.start('Executing postBuild()'); - await executePluginsPostBuild({plugins, props, collectedData}); - PerfLogger.end('Executing postBuild()'); + await PerfLogger.async('postBuild()', () => + executePluginsPostBuild({plugins, props, collectedData}), + ); // TODO execute this in parallel to postBuild? - PerfLogger.start('Executing broken links checker'); - await executeBrokenLinksCheck({props, collectedData}); - PerfLogger.end('Executing broken links checker'); + await PerfLogger.async('Broken links checker', () => + executeBrokenLinksCheck({props, collectedData}), + ); logger.success`Generated static files in path=${path.relative( process.cwd(), outDir, )}.`; - if (isLastLocale) { - logger.info`Use code=${'npm run serve'} command to test your build locally.`; - } - - if (forceTerminate && isLastLocale && !cliOptions.bundleAnalyzer) { - process.exit(0); - } - return outDir; } @@ -247,40 +228,39 @@ async function executeSSG({ serverBundlePath: string; clientManifestPath: string; }) { - PerfLogger.start('Reading client manifest'); - const manifest: Manifest = await fs.readJSON(clientManifestPath, 'utf-8'); - PerfLogger.end('Reading client manifest'); + const manifest: Manifest = await PerfLogger.async( + 'Read client manifest', + () => fs.readJSON(clientManifestPath, 'utf-8'), + ); - PerfLogger.start('Compiling SSR template'); - const ssrTemplate = await compileSSRTemplate( - props.siteConfig.ssrTemplate ?? defaultSSRTemplate, + const ssrTemplate = await PerfLogger.async('Compile SSR template', () => + compileSSRTemplate(props.siteConfig.ssrTemplate ?? defaultSSRTemplate), ); - PerfLogger.end('Compiling SSR template'); - PerfLogger.start('Loading App renderer'); - const renderer = await loadAppRenderer({ - serverBundlePath, - }); - PerfLogger.end('Loading App renderer'); - - PerfLogger.start('Generate static files'); - const ssgResult = await generateStaticFiles({ - pathnames: props.routesPaths, - renderer, - params: { - trailingSlash: props.siteConfig.trailingSlash, - outDir: props.outDir, - baseUrl: props.baseUrl, - manifest, - headTags: props.headTags, - preBodyTags: props.preBodyTags, - postBodyTags: props.postBodyTags, - ssrTemplate, - noIndex: props.siteConfig.noIndex, - DOCUSAURUS_VERSION, - }, - }); - PerfLogger.end('Generate static files'); + const renderer = await PerfLogger.async('Load App renderer', () => + loadAppRenderer({ + serverBundlePath, + }), + ); + + const ssgResult = await PerfLogger.async('Generate static files', () => + generateStaticFiles({ + pathnames: props.routesPaths, + renderer, + params: { + trailingSlash: props.siteConfig.trailingSlash, + outDir: props.outDir, + baseUrl: props.baseUrl, + manifest, + headTags: props.headTags, + preBodyTags: props.preBodyTags, + postBodyTags: props.postBodyTags, + ssrTemplate, + noIndex: props.siteConfig.noIndex, + DOCUSAURUS_VERSION, + }, + }), + ); return ssgResult; } diff --git a/packages/docusaurus/src/commands/start/utils.ts b/packages/docusaurus/src/commands/start/utils.ts index f23ac13ecfc6..1d1c70766132 100644 --- a/packages/docusaurus/src/commands/start/utils.ts +++ b/packages/docusaurus/src/commands/start/utils.ts @@ -18,6 +18,7 @@ import { reloadSite, reloadSitePlugin, } from '../../server/site'; +import {formatPluginName} from '../../server/plugins/pluginsUtils'; import type {StartCLIOptions} from './start'; import type {LoadedPlugin} from '@docusaurus/types'; @@ -69,10 +70,13 @@ async function createLoadSiteParams({ export async function createReloadableSite(startParams: StartParams) { const openUrlContext = await createOpenUrlContext(startParams); - let site = await PerfLogger.async('Loading site', async () => { - const params = await createLoadSiteParams(startParams); - return loadSite(params); - }); + const loadSiteParams = await PerfLogger.async('createLoadSiteParams', () => + createLoadSiteParams(startParams), + ); + + let site = await PerfLogger.async('Load site', () => + loadSite(loadSiteParams), + ); const get = () => site; @@ -89,7 +93,7 @@ export async function createReloadableSite(startParams: StartParams) { const reloadBase = async () => { try { const oldSite = site; - site = await PerfLogger.async('Reloading site', () => reloadSite(site)); + site = await PerfLogger.async('Reload site', () => reloadSite(site)); if (oldSite.props.baseUrl !== site.props.baseUrl) { printOpenUrlMessage(); } @@ -108,7 +112,7 @@ export async function createReloadableSite(startParams: StartParams) { const reloadPlugin = async (plugin: LoadedPlugin) => { try { site = await PerfLogger.async( - `Reloading site plugin ${plugin.name}@${plugin.options.id}`, + `Reload site plugin ${formatPluginName(plugin)}`, () => { const pluginIdentifier = {name: plugin.name, id: plugin.options.id}; return reloadSitePlugin(site, pluginIdentifier); @@ -116,7 +120,7 @@ export async function createReloadableSite(startParams: StartParams) { ); } catch (e) { logger.error( - `Site plugin reload failure - Plugin ${plugin.name}@${plugin.options.id}`, + `Site plugin reload failure - Plugin ${formatPluginName(plugin)}`, ); console.error(e); } diff --git a/packages/docusaurus/src/commands/writeTranslations.ts b/packages/docusaurus/src/commands/writeTranslations.ts index 9b94cedae28c..9460622c7bef 100644 --- a/packages/docusaurus/src/commands/writeTranslations.ts +++ b/packages/docusaurus/src/commands/writeTranslations.ts @@ -13,7 +13,7 @@ import { writePluginTranslations, writeCodeTranslations, type WriteTranslationsOptions, - getPluginsDefaultCodeTranslationMessages, + loadPluginsDefaultCodeTranslationMessages, applyDefaultCodeTranslations, } from '../server/translations/translations'; import { @@ -114,7 +114,7 @@ Available locales are: ${context.i18n.locales.join(',')}.`, await getExtraSourceCodeFilePaths(), ); - const defaultCodeMessages = await getPluginsDefaultCodeTranslationMessages( + const defaultCodeMessages = await loadPluginsDefaultCodeTranslationMessages( plugins, ); diff --git a/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap b/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap index e551f0278426..8a48af985e45 100644 --- a/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap +++ b/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap @@ -36,6 +36,7 @@ exports[`load loads props for site with custom i18n path 1`] = ` "plugins": [ { "content": undefined, + "defaultCodeTranslations": {}, "getClientModules": [Function], "globalData": undefined, "injectHtmlTags": [Function], @@ -52,6 +53,7 @@ exports[`load loads props for site with custom i18n path 1`] = ` { "configureWebpack": [Function], "content": undefined, + "defaultCodeTranslations": {}, "globalData": undefined, "name": "docusaurus-mdx-fallback-plugin", "options": { @@ -132,5 +134,6 @@ exports[`load loads props for site with custom i18n path 1`] = ` "pluginVersions": {}, "siteVersion": undefined, }, + "siteVersion": undefined, } `; diff --git a/packages/docusaurus/src/server/__tests__/clientModules.test.ts b/packages/docusaurus/src/server/__tests__/clientModules.test.ts index 5e22efd63057..cc358edfa106 100644 --- a/packages/docusaurus/src/server/__tests__/clientModules.test.ts +++ b/packages/docusaurus/src/server/__tests__/clientModules.test.ts @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import {loadClientModules} from '../clientModules'; +import {getAllClientModules} from '../clientModules'; import type {LoadedPlugin} from '@docusaurus/types'; const pluginEmpty = { @@ -33,14 +33,14 @@ const pluginHelloWorld = { }, } as unknown as LoadedPlugin; -describe('loadClientModules', () => { +describe('getAllClientModules', () => { it('loads an empty plugin', () => { - const clientModules = loadClientModules([pluginEmpty]); + const clientModules = getAllClientModules([pluginEmpty]); expect(clientModules).toMatchInlineSnapshot(`[]`); }); it('loads a non-empty plugin', () => { - const clientModules = loadClientModules([pluginFooBar]); + const clientModules = getAllClientModules([pluginFooBar]); expect(clientModules).toMatchInlineSnapshot(` [ "/packages/docusaurus/src/server/__tests__/foo", @@ -50,7 +50,7 @@ describe('loadClientModules', () => { }); it('loads multiple non-empty plugins', () => { - const clientModules = loadClientModules([pluginFooBar, pluginHelloWorld]); + const clientModules = getAllClientModules([pluginFooBar, pluginHelloWorld]); expect(clientModules).toMatchInlineSnapshot(` [ "/packages/docusaurus/src/server/__tests__/foo", @@ -62,7 +62,7 @@ describe('loadClientModules', () => { }); it('loads multiple non-empty plugins in different order', () => { - const clientModules = loadClientModules([pluginHelloWorld, pluginFooBar]); + const clientModules = getAllClientModules([pluginHelloWorld, pluginFooBar]); expect(clientModules).toMatchInlineSnapshot(` [ "/hello", @@ -74,7 +74,7 @@ describe('loadClientModules', () => { }); it('loads both empty and non-empty plugins', () => { - const clientModules = loadClientModules([ + const clientModules = getAllClientModules([ pluginHelloWorld, pluginEmpty, pluginFooBar, @@ -90,7 +90,7 @@ describe('loadClientModules', () => { }); it('loads empty and non-empty in a different order', () => { - const clientModules = loadClientModules([ + const clientModules = getAllClientModules([ pluginHelloWorld, pluginFooBar, pluginEmpty, diff --git a/packages/docusaurus/src/server/__tests__/siteMetadata.test.ts b/packages/docusaurus/src/server/__tests__/siteMetadata.test.ts index cf5557ecb496..3ebfacf325fb 100644 --- a/packages/docusaurus/src/server/__tests__/siteMetadata.test.ts +++ b/packages/docusaurus/src/server/__tests__/siteMetadata.test.ts @@ -7,13 +7,13 @@ import path from 'path'; import {DOCUSAURUS_VERSION} from '@docusaurus/utils'; -import {getPluginVersion, loadSiteMetadata} from '../siteMetadata'; +import {loadPluginVersion, createSiteMetadata} from '../siteMetadata'; import type {LoadedPlugin} from '@docusaurus/types'; -describe('getPluginVersion', () => { +describe('loadPluginVersion', () => { it('detects external packages plugins versions', async () => { await expect( - getPluginVersion( + loadPluginVersion( path.join(__dirname, '__fixtures__/siteMetadata/dummy-plugin.js'), // Make the plugin appear external. path.join(__dirname, '..', '..', '..', '..', '..', '..', 'website'), @@ -23,7 +23,7 @@ describe('getPluginVersion', () => { it('detects project plugins versions', async () => { await expect( - getPluginVersion( + loadPluginVersion( path.join(__dirname, '__fixtures__/siteMetadata/dummy-plugin.js'), // Make the plugin appear project local. path.join(__dirname, '__fixtures__/siteMetadata'), @@ -32,14 +32,14 @@ describe('getPluginVersion', () => { }); it('detects local packages versions', async () => { - await expect(getPluginVersion('/', '/')).resolves.toEqual({type: 'local'}); + await expect(loadPluginVersion('/', '/')).resolves.toEqual({type: 'local'}); }); }); -describe('loadSiteMetadata', () => { - it('throws if plugin versions mismatch', async () => { - await expect( - loadSiteMetadata({ +describe('createSiteMetadata', () => { + it('throws if plugin versions mismatch', () => { + expect(() => + createSiteMetadata({ plugins: [ { name: 'docusaurus-plugin-content-docs', @@ -50,10 +50,9 @@ describe('loadSiteMetadata', () => { }, }, ] as LoadedPlugin[], - siteDir: path.join(__dirname, '__fixtures__/siteMetadata'), + siteVersion: 'some-random-version', }), - ).rejects - .toThrow(`Invalid name=docusaurus-plugin-content-docs version number=1.0.0. + ).toThrow(`Invalid name=docusaurus-plugin-content-docs version number=1.0.0. All official @docusaurus/* packages should have the exact same version as @docusaurus/core (number=${DOCUSAURUS_VERSION}). Maybe you want to check, or regenerate your yarn.lock or package-lock.json file?`); }); diff --git a/packages/docusaurus/src/server/clientModules.ts b/packages/docusaurus/src/server/clientModules.ts index 5a2057fe611c..bdb946430529 100644 --- a/packages/docusaurus/src/server/clientModules.ts +++ b/packages/docusaurus/src/server/clientModules.ts @@ -12,7 +12,7 @@ import type {LoadedPlugin} from '@docusaurus/types'; * Runs the `getClientModules` lifecycle. The returned file paths are all * absolute. */ -export function loadClientModules(plugins: LoadedPlugin[]): string[] { +export function getAllClientModules(plugins: LoadedPlugin[]): string[] { return plugins.flatMap( (plugin) => plugin.getClientModules?.().map((p) => path.resolve(plugin.path, p)) ?? diff --git a/packages/docusaurus/src/server/plugins/actions.ts b/packages/docusaurus/src/server/plugins/actions.ts index 3e7d389a90ca..d9313cb4cccd 100644 --- a/packages/docusaurus/src/server/plugins/actions.ts +++ b/packages/docusaurus/src/server/plugins/actions.ts @@ -48,6 +48,7 @@ export async function createPluginActionsUtils({ dataDir, `${docuHash('pluginRouteContextModule')}.json`, ); + // TODO not ideal place to generate that file await generate( '/', pluginRouteContextModulePath, diff --git a/packages/docusaurus/src/server/plugins/init.ts b/packages/docusaurus/src/server/plugins/init.ts index c19f82d15b80..a6100c24ec59 100644 --- a/packages/docusaurus/src/server/plugins/init.ts +++ b/packages/docusaurus/src/server/plugins/init.ts @@ -12,7 +12,7 @@ import { normalizePluginOptions, normalizeThemeConfig, } from '@docusaurus/utils-validation'; -import {getPluginVersion} from '../siteMetadata'; +import {loadPluginVersion} from '../siteMetadata'; import {ensureUniquePluginInstanceIds} from './pluginIds'; import {loadPluginConfigs, type NormalizedPluginConfig} from './configs'; import type { @@ -61,14 +61,14 @@ export async function initPlugins( const pluginRequire = createRequire(context.siteConfigPath); const pluginConfigs = await loadPluginConfigs(context); - async function doGetPluginVersion( + async function doLoadPluginVersion( normalizedPluginConfig: NormalizedPluginConfig, ): Promise { if (normalizedPluginConfig.pluginModule?.path) { const pluginPath = pluginRequire.resolve( normalizedPluginConfig.pluginModule.path, ); - return getPluginVersion(pluginPath, context.siteDir); + return loadPluginVersion(pluginPath, context.siteDir); } return {type: 'local'}; } @@ -109,7 +109,7 @@ export async function initPlugins( async function initializePlugin( normalizedPluginConfig: NormalizedPluginConfig, ): Promise { - const pluginVersion: PluginVersionInformation = await doGetPluginVersion( + const pluginVersion: PluginVersionInformation = await doLoadPluginVersion( normalizedPluginConfig, ); const pluginOptions = doValidatePluginOptions(normalizedPluginConfig); diff --git a/packages/docusaurus/src/server/plugins/plugins.ts b/packages/docusaurus/src/server/plugins/plugins.ts index c08dbcdb2333..a5dbf2633892 100644 --- a/packages/docusaurus/src/server/plugins/plugins.ts +++ b/packages/docusaurus/src/server/plugins/plugins.ts @@ -15,6 +15,7 @@ import { aggregateAllContent, aggregateGlobalData, aggregateRoutes, + formatPluginName, getPluginByIdentifier, mergeGlobalData, } from './pluginsUtils'; @@ -73,46 +74,57 @@ async function executePluginContentLoading({ plugin: InitializedPlugin; context: LoadContext; }): Promise { - return PerfLogger.async( - `Plugins - single plugin content loading - ${plugin.name}@${plugin.options.id}`, - async () => { - let content = await plugin.loadContent?.(); + return PerfLogger.async(`Load ${formatPluginName(plugin)}`, async () => { + let content = await PerfLogger.async('loadContent()', () => + plugin.loadContent?.(), + ); - content = await translatePluginContent({ + content = await PerfLogger.async('translatePluginContent()', () => + translatePluginContent({ plugin, content, context, - }); - - if (!plugin.contentLoaded) { - return { - ...plugin, - content, - routes: [], - globalData: undefined, - }; - } - - const pluginActionsUtils = await createPluginActionsUtils({ - plugin, - generatedFilesDir: context.generatedFilesDir, - baseUrl: context.siteConfig.baseUrl, - trailingSlash: context.siteConfig.trailingSlash, - }); + }), + ); - await plugin.contentLoaded({ - content, - actions: pluginActionsUtils.getActions(), - }); + const defaultCodeTranslations = + (await PerfLogger.async('getDefaultCodeTranslationMessages()', () => + plugin.getDefaultCodeTranslationMessages?.(), + )) ?? {}; + if (!plugin.contentLoaded) { return { ...plugin, content, - routes: pluginActionsUtils.getRoutes(), - globalData: pluginActionsUtils.getGlobalData(), + defaultCodeTranslations, + routes: [], + globalData: undefined, }; - }, - ); + } + + const pluginActionsUtils = await createPluginActionsUtils({ + plugin, + generatedFilesDir: context.generatedFilesDir, + baseUrl: context.siteConfig.baseUrl, + trailingSlash: context.siteConfig.trailingSlash, + }); + + await PerfLogger.async('contentLoaded()', () => + // @ts-expect-error: should autofix with TS 5.4 + plugin.contentLoaded({ + content, + actions: pluginActionsUtils.getActions(), + }), + ); + + return { + ...plugin, + content, + defaultCodeTranslations, + routes: pluginActionsUtils.getRoutes(), + globalData: pluginActionsUtils.getGlobalData(), + }; + }); } async function executeAllPluginsContentLoading({ @@ -122,7 +134,7 @@ async function executeAllPluginsContentLoading({ plugins: InitializedPlugin[]; context: LoadContext; }): Promise { - return PerfLogger.async(`Plugins - all plugins content loading`, () => { + return PerfLogger.async(`Load plugins content`, () => { return Promise.all( plugins.map((plugin) => executePluginContentLoading({plugin, context})), ); @@ -139,7 +151,7 @@ async function executePluginAllContentLoaded({ allContent: AllContent; }): Promise<{routes: RouteConfig[]; globalData: unknown}> { return PerfLogger.async( - `Plugins - allContentLoaded - ${plugin.name}@${plugin.options.id}`, + `allContentLoaded() - ${formatPluginName(plugin)}`, async () => { if (!plugin.allContentLoaded) { return {routes: [], globalData: undefined}; @@ -171,7 +183,7 @@ async function executeAllPluginsAllContentLoaded({ plugins: LoadedPlugin[]; context: LoadContext; }): Promise { - return PerfLogger.async(`Plugins - allContentLoaded`, async () => { + return PerfLogger.async(`allContentLoaded()`, async () => { const allContent = aggregateAllContent(plugins); const routes: RouteConfig[] = []; @@ -199,6 +211,9 @@ async function executeAllPluginsAllContentLoaded({ }); } +// This merges plugins routes and global data created from both lifecycles: +// - contentLoaded() +// - allContentLoaded() function mergeResults({ plugins, allContentLoadedResult, @@ -232,9 +247,9 @@ export type LoadPluginsResult = { export async function loadPlugins( context: LoadContext, ): Promise { - return PerfLogger.async('Plugins - loadPlugins', async () => { + return PerfLogger.async('Load plugins', async () => { const initializedPlugins: InitializedPlugin[] = await PerfLogger.async( - 'Plugins - initPlugins', + 'Init plugins', () => initPlugins(context), ); @@ -272,36 +287,39 @@ export async function reloadPlugin({ plugins: LoadedPlugin[]; context: LoadContext; }): Promise { - return PerfLogger.async('Plugins - reloadPlugin', async () => { - const previousPlugin = getPluginByIdentifier({ - plugins: previousPlugins, - pluginIdentifier, - }); - const plugin = await executePluginContentLoading({ - plugin: previousPlugin, - context, - }); + return PerfLogger.async( + `Reload plugin ${formatPluginName(pluginIdentifier)}`, + async () => { + const previousPlugin = getPluginByIdentifier({ + plugins: previousPlugins, + pluginIdentifier, + }); + const plugin = await executePluginContentLoading({ + plugin: previousPlugin, + context, + }); - /* + /* // TODO Docusaurus v4 - upgrade to Node 20, use array.with() const plugins = previousPlugins.with( previousPlugins.indexOf(previousPlugin), plugin, ); */ - const plugins = [...previousPlugins]; - plugins[previousPlugins.indexOf(previousPlugin)] = plugin; + const plugins = [...previousPlugins]; + plugins[previousPlugins.indexOf(previousPlugin)] = plugin; - const allContentLoadedResult = await executeAllPluginsAllContentLoaded({ - plugins, - context, - }); + const allContentLoadedResult = await executeAllPluginsAllContentLoaded({ + plugins, + context, + }); - const {routes, globalData} = mergeResults({ - plugins, - allContentLoadedResult, - }); + const {routes, globalData} = mergeResults({ + plugins, + allContentLoadedResult, + }); - return {plugins, routes, globalData}; - }); + return {plugins, routes, globalData}; + }, + ); } diff --git a/packages/docusaurus/src/server/plugins/pluginsUtils.ts b/packages/docusaurus/src/server/plugins/pluginsUtils.ts index 706e6ecafa06..4c756fd7fdb5 100644 --- a/packages/docusaurus/src/server/plugins/pluginsUtils.ts +++ b/packages/docusaurus/src/server/plugins/pluginsUtils.ts @@ -29,7 +29,9 @@ export function getPluginByIdentifier

({ ); if (!plugin) { throw new Error( - logger.interpolate`Plugin not found for identifier ${pluginIdentifier.name}@${pluginIdentifier.id}`, + logger.interpolate`Plugin not found for identifier ${formatPluginName( + pluginIdentifier, + )}`, ); } return plugin; @@ -85,3 +87,22 @@ export function mergeGlobalData(...globalDataList: GlobalData[]): GlobalData { return result; } + +// This is primarily useful for colored logging purpose +// Do not rely on this for logic +export function formatPluginName( + plugin: InitializedPlugin | PluginIdentifier, +): string { + let formattedName = plugin.name; + // Hacky way to reduce string size for logging purpose + formattedName = formattedName.replace('docusaurus-plugin-content-', ''); + formattedName = formattedName.replace('docusaurus-plugin-', ''); + formattedName = formattedName.replace('docusaurus-theme-', ''); + formattedName = formattedName.replace('-plugin', ''); + formattedName = logger.name(formattedName); + + const id = 'id' in plugin ? plugin.id : plugin.options.id; + const formattedId = logger.subdue(id); + + return `${formattedName}@${formattedId}`; +} diff --git a/packages/docusaurus/src/server/site.ts b/packages/docusaurus/src/server/site.ts index c693974fbeba..55dcb743c8a8 100644 --- a/packages/docusaurus/src/server/site.ts +++ b/packages/docusaurus/src/server/site.ts @@ -13,14 +13,14 @@ import { } from '@docusaurus/utils'; import combinePromises from 'combine-promises'; import {loadSiteConfig} from './config'; -import {loadClientModules} from './clientModules'; +import {getAllClientModules} from './clientModules'; import {loadPlugins, reloadPlugin} from './plugins/plugins'; import {loadHtmlTags} from './htmlTags'; -import {loadSiteMetadata} from './siteMetadata'; +import {createSiteMetadata, loadSiteVersion} from './siteMetadata'; import {loadI18n} from './i18n'; import { loadSiteCodeTranslations, - getPluginsDefaultCodeTranslationMessages, + getPluginsDefaultCodeTranslations, } from './translations/translations'; import {PerfLogger} from '../utils'; import {generateSiteFiles} from './codegen/codegen'; @@ -76,9 +76,15 @@ export async function loadContext( } = params; const generatedFilesDir = path.resolve(siteDir, GENERATED_FILES_DIR_NAME); - const {siteConfig: initialSiteConfig, siteConfigPath} = await loadSiteConfig({ - siteDir, - customConfigFilePath, + const { + siteVersion, + loadSiteConfig: {siteConfig: initialSiteConfig, siteConfigPath}, + } = await combinePromises({ + siteVersion: loadSiteVersion(siteDir), + loadSiteConfig: loadSiteConfig({ + siteDir, + customConfigFilePath, + }), }); const i18n = await loadI18n(initialSiteConfig, {locale}); @@ -107,6 +113,7 @@ export async function loadContext( return { siteDir, + siteVersion, generatedFilesDir, localizationDir, siteConfig, @@ -118,13 +125,14 @@ export async function loadContext( }; } -async function createSiteProps( +function createSiteProps( params: LoadPluginsResult & {context: LoadContext}, -): Promise { +): Props { const {plugins, routes, context} = params; const { generatedFilesDir, siteDir, + siteVersion, siteConfig, siteConfigPath, outDir, @@ -136,19 +144,12 @@ async function createSiteProps( const {headTags, preBodyTags, postBodyTags} = loadHtmlTags(plugins); - const {codeTranslations, siteMetadata} = await combinePromises({ - // TODO code translations should be loaded as part of LoadedPlugin? - codeTranslations: PerfLogger.async( - 'Load - loadCodeTranslations', - async () => ({ - ...(await getPluginsDefaultCodeTranslationMessages(plugins)), - ...siteCodeTranslations, - }), - ), - siteMetadata: PerfLogger.async('Load - loadSiteMetadata', () => - loadSiteMetadata({plugins, siteDir}), - ), - }); + const siteMetadata = createSiteMetadata({plugins, siteVersion}); + + const codeTranslations = { + ...getPluginsDefaultCodeTranslations({plugins}), + ...siteCodeTranslations, + }; handleDuplicateRoutes(routes, siteConfig.onDuplicateRoutes); const routesPaths = getRoutesPaths(routes, baseUrl); @@ -157,6 +158,7 @@ async function createSiteProps( siteConfig, siteConfigPath, siteMetadata, + siteVersion, siteDir, outDir, baseUrl, @@ -181,7 +183,7 @@ async function createSiteFiles({ site: Site; globalData: GlobalData; }) { - return PerfLogger.async('Load - createSiteFiles', async () => { + return PerfLogger.async('Create site files', async () => { const { props: { plugins, @@ -194,7 +196,7 @@ async function createSiteFiles({ baseUrl, }, } = site; - const clientModules = loadClientModules(plugins); + const clientModules = getAllClientModules(plugins); await generateSiteFiles({ generatedFilesDir, clientModules, @@ -216,13 +218,11 @@ async function createSiteFiles({ * it generates temp files in the `.docusaurus` folder for the bundler. */ export async function loadSite(params: LoadContextParams): Promise { - PerfLogger.start('Load - loadContext'); - const context = await loadContext(params); - PerfLogger.end('Load - loadContext'); + const context = await PerfLogger.async('Load context', () => + loadContext(params), + ); - PerfLogger.start('Load - loadPlugins'); const {plugins, routes, globalData} = await loadPlugins(context); - PerfLogger.end('Load - loadPlugins'); const props = await createSiteProps({plugins, routes, globalData, context}); diff --git a/packages/docusaurus/src/server/siteMetadata.ts b/packages/docusaurus/src/server/siteMetadata.ts index faedab4fcfdb..3c52df408725 100644 --- a/packages/docusaurus/src/server/siteMetadata.ts +++ b/packages/docusaurus/src/server/siteMetadata.ts @@ -14,7 +14,7 @@ import type { SiteMetadata, } from '@docusaurus/types'; -async function getPackageJsonVersion( +async function loadPackageJsonVersion( packageJsonPath: string, ): Promise { if (await fs.pathExists(packageJsonPath)) { @@ -24,14 +24,20 @@ async function getPackageJsonVersion( return undefined; } -async function getPackageJsonName( +async function loadPackageJsonName( packageJsonPath: string, ): Promise { // eslint-disable-next-line @typescript-eslint/no-var-requires, import/no-dynamic-require, global-require return (require(packageJsonPath) as {name?: string}).name; } -export async function getPluginVersion( +export async function loadSiteVersion( + siteDir: string, +): Promise { + return loadPackageJsonVersion(path.join(siteDir, 'package.json')); +} + +export async function loadPluginVersion( pluginPath: string, siteDir: string, ): Promise { @@ -52,8 +58,8 @@ export async function getPluginVersion( } return { type: 'package', - name: await getPackageJsonName(packageJsonPath), - version: await getPackageJsonVersion(packageJsonPath), + name: await loadPackageJsonName(packageJsonPath), + version: await loadPackageJsonVersion(packageJsonPath), }; } potentialPluginPackageJsonDirectory = path.dirname( @@ -89,18 +95,16 @@ Maybe you want to check, or regenerate your yarn.lock or package-lock.json file? ); } -export async function loadSiteMetadata({ +export function createSiteMetadata({ + siteVersion, plugins, - siteDir, }: { + siteVersion: string | undefined; plugins: LoadedPlugin[]; - siteDir: string; -}): Promise { +}): SiteMetadata { const siteMetadata: SiteMetadata = { docusaurusVersion: DOCUSAURUS_VERSION, - siteVersion: await getPackageJsonVersion( - path.join(siteDir, 'package.json'), - ), + siteVersion, pluginVersions: Object.fromEntries( plugins .filter(({version: {type}}) => type !== 'synthetic') diff --git a/packages/docusaurus/src/server/translations/__tests__/translations.test.ts b/packages/docusaurus/src/server/translations/__tests__/translations.test.ts index 230031766b88..7d8253dfa351 100644 --- a/packages/docusaurus/src/server/translations/__tests__/translations.test.ts +++ b/packages/docusaurus/src/server/translations/__tests__/translations.test.ts @@ -15,7 +15,7 @@ import { readCodeTranslationFileContent, type WriteTranslationsOptions, localizePluginTranslationFile, - getPluginsDefaultCodeTranslationMessages, + loadPluginsDefaultCodeTranslationMessages, applyDefaultCodeTranslations, } from '../translations'; import type { @@ -537,7 +537,7 @@ describe('readCodeTranslationFileContent', () => { }); }); -describe('getPluginsDefaultCodeTranslationMessages', () => { +describe('loadPluginsDefaultCodeTranslationMessages', () => { function createTestPlugin( fn: InitializedPlugin['getDefaultCodeTranslationMessages'], ): InitializedPlugin { @@ -547,14 +547,14 @@ describe('getPluginsDefaultCodeTranslationMessages', () => { it('works for empty plugins', async () => { const plugins: InitializedPlugin[] = []; await expect( - getPluginsDefaultCodeTranslationMessages(plugins), + loadPluginsDefaultCodeTranslationMessages(plugins), ).resolves.toEqual({}); }); it('works for 1 plugin without lifecycle', async () => { const plugins: InitializedPlugin[] = [createTestPlugin(undefined)]; await expect( - getPluginsDefaultCodeTranslationMessages(plugins), + loadPluginsDefaultCodeTranslationMessages(plugins), ).resolves.toEqual({}); }); @@ -566,7 +566,7 @@ describe('getPluginsDefaultCodeTranslationMessages', () => { })), ]; await expect( - getPluginsDefaultCodeTranslationMessages(plugins), + loadPluginsDefaultCodeTranslationMessages(plugins), ).resolves.toEqual({ a: '1', b: '2', @@ -585,7 +585,7 @@ describe('getPluginsDefaultCodeTranslationMessages', () => { })), ]; await expect( - getPluginsDefaultCodeTranslationMessages(plugins), + loadPluginsDefaultCodeTranslationMessages(plugins), ).resolves.toEqual({ a: '1', b: '2', @@ -613,7 +613,7 @@ describe('getPluginsDefaultCodeTranslationMessages', () => { createTestPlugin(undefined), ]; await expect( - getPluginsDefaultCodeTranslationMessages(plugins), + loadPluginsDefaultCodeTranslationMessages(plugins), ).resolves.toEqual({ // merge, last plugin wins b: '2', diff --git a/packages/docusaurus/src/server/translations/translations.ts b/packages/docusaurus/src/server/translations/translations.ts index ceb4efc105ae..51140f33477f 100644 --- a/packages/docusaurus/src/server/translations/translations.ts +++ b/packages/docusaurus/src/server/translations/translations.ts @@ -20,6 +20,7 @@ import type { TranslationFile, CodeTranslations, InitializedPlugin, + LoadedPlugin, } from '@docusaurus/types'; export type WriteTranslationsOptions = { @@ -242,17 +243,33 @@ export async function localizePluginTranslationFile({ return translationFile; } -export async function getPluginsDefaultCodeTranslationMessages( +export function mergeCodeTranslations( + codeTranslations: CodeTranslations[], +): CodeTranslations { + return codeTranslations.reduce( + (allCodeTranslations, current) => ({ + ...allCodeTranslations, + ...current, + }), + {}, + ); +} + +export async function loadPluginsDefaultCodeTranslationMessages( plugins: InitializedPlugin[], ): Promise { const pluginsMessages = await Promise.all( plugins.map((plugin) => plugin.getDefaultCodeTranslationMessages?.() ?? {}), ); + return mergeCodeTranslations(pluginsMessages); +} - return pluginsMessages.reduce( - (allMessages, pluginMessages) => ({...allMessages, ...pluginMessages}), - {}, - ); +export function getPluginsDefaultCodeTranslations({ + plugins, +}: { + plugins: LoadedPlugin[]; +}): CodeTranslations { + return mergeCodeTranslations(plugins.map((p) => p.defaultCodeTranslations)); } export function applyDefaultCodeTranslations({ diff --git a/packages/docusaurus/src/ssg.ts b/packages/docusaurus/src/ssg.ts index a1e6d4e9ce18..9fc0cb7d66c0 100644 --- a/packages/docusaurus/src/ssg.ts +++ b/packages/docusaurus/src/ssg.ts @@ -47,12 +47,11 @@ export async function loadAppRenderer({ }: { serverBundlePath: string; }): Promise { - console.log(`SSG - Load server bundle`); - PerfLogger.start(`SSG - Load server bundle`); - const source = await fs.readFile(serverBundlePath); - PerfLogger.end(`SSG - Load server bundle`); + const source = await PerfLogger.async(`Load server bundle`, () => + fs.readFile(serverBundlePath), + ); PerfLogger.log( - `SSG - Server bundle size = ${(source.length / 1024000).toFixed(3)} MB`, + `Server bundle size = ${(source.length / 1024000).toFixed(3)} MB`, ); const filename = path.basename(serverBundlePath); @@ -69,14 +68,16 @@ export async function loadAppRenderer({ require: createRequire(serverBundlePath), }; - PerfLogger.start(`SSG - Evaluate server bundle`); - const serverEntry = evaluate( - source, - /* filename: */ filename, - /* scope: */ globals, - /* includeGlobals: */ true, - ) as {default?: AppRenderer}; - PerfLogger.end(`SSG - Evaluate server bundle`); + const serverEntry = await PerfLogger.async( + `Evaluate server bundle`, + () => + evaluate( + source, + /* filename: */ filename, + /* scope: */ globals, + /* includeGlobals: */ true, + ) as {default?: AppRenderer}, + ); if (!serverEntry?.default || typeof serverEntry.default !== 'function') { throw new Error( diff --git a/packages/docusaurus/src/utils.ts b/packages/docusaurus/src/utils.ts index 660072bae298..59bab6325a6e 100644 --- a/packages/docusaurus/src/utils.ts +++ b/packages/docusaurus/src/utils.ts @@ -4,6 +4,7 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. */ +import {AsyncLocalStorage} from 'async_hooks'; import logger from '@docusaurus/logger'; // For now this is a private env variable we use internally @@ -17,6 +18,16 @@ const Thresholds = { red: 1000, }; +const PerfPrefix = logger.yellow(`[PERF] `); + +// This is what enables to "see the parent stack" for each log +// Parent1 > Parent2 > Parent3 > child trace +const ParentPrefix = new AsyncLocalStorage(); +function applyParentPrefix(label: string) { + const parentPrefix = ParentPrefix.getStore(); + return parentPrefix ? `${parentPrefix} > ${label}` : label; +} + type PerfLoggerAPI = { start: (label: string) => void; end: (label: string) => void; @@ -38,8 +49,6 @@ function createPerfLogger(): PerfLoggerAPI { }; } - const prefix = logger.yellow(`[PERF] `); - const formatDuration = (duration: number): string => { if (duration > Thresholds.red) { return logger.red(`${(duration / 1000).toFixed(2)} seconds!`); @@ -54,7 +63,7 @@ function createPerfLogger(): PerfLoggerAPI { if (duration < Thresholds.min) { return; } - console.log(`${prefix + label} - ${formatDuration(duration)}`); + console.log(`${PerfPrefix + label} - ${formatDuration(duration)}`); }; const start: PerfLoggerAPI['start'] = (label) => performance.mark(label); @@ -62,18 +71,18 @@ function createPerfLogger(): PerfLoggerAPI { const end: PerfLoggerAPI['end'] = (label) => { const {duration} = performance.measure(label); performance.clearMarks(label); - logDuration(label, duration); + logDuration(applyParentPrefix(label), duration); }; const log: PerfLoggerAPI['log'] = (label: string) => - console.log(prefix + label); + console.log(PerfPrefix + applyParentPrefix(label)); const async: PerfLoggerAPI['async'] = async (label, asyncFn) => { - start(label); + const finalLabel = applyParentPrefix(label); const before = performance.now(); - const result = await asyncFn(); + const result = await ParentPrefix.run(finalLabel, () => asyncFn()); const duration = performance.now() - before; - logDuration(label, duration); + logDuration(finalLabel, duration); return result; }; diff --git a/project-words.txt b/project-words.txt index 01f35407ec24..503af3e1edba 100644 --- a/project-words.txt +++ b/project-words.txt @@ -16,6 +16,7 @@ architecting Astro atrule Autoconverted +autofix Autogen autogen autogenerating