Add entrypoint tracing (#25538)

This adds tracing entrypoints directly after they have have been transpiled to allow us to trace before the webpack runtime has been added to the modules. This should allow for more accurate tracing of entrypoints and allow the trace step to be cached. 

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added


x-ref: #24700
x-ref: #26200
x-ref: #23894
x-ref: #25431

Author: ijjk

packages/next/build/index.ts

@@ -660,6 +660,8 @@ export default async function build(
     const serverPropsPages = new Set<string>()
     const additionalSsgPaths = new Map<string, Array<string>>()
     const additionalSsgPathsEncoded = new Map<string, Array<string>>()
+    const pageTraceIncludes = new Map<string, Array<string>>()
+    const pageTraceExcludes = new Map<string, Array<string>>()
     const pageInfos = new Map<string, PageInfo>()
     const pagesManifest = JSON.parse(
       await promises.readFile(manifestPath, 'utf8')
@@ -841,6 +843,11 @@ export default async function build(
                   )
                 })
 
+                if (config.experimental.nftTracing) {
+                  pageTraceIncludes.set(page, workerResult.traceIncludes || [])
+                  pageTraceExcludes.set(page, workerResult.traceExcludes || [])
+                }
+
                 if (
                   workerResult.isStatic === false &&
                   (workerResult.isHybridAmp || workerResult.isAmpOnly)
@@ -979,6 +986,63 @@ export default async function build(
       )
     }
 
+    if (config.experimental.nftTracing) {
+      const globOrig = require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob')
+      const glob = (pattern: string): Promise<string[]> => {
+        return new Promise((resolve, reject) => {
+          globOrig(pattern, { cwd: dir }, (err, files) => {
+            if (err) {
+              return reject(err)
+            }
+            resolve(files)
+          })
+        })
+      }
+
+      for (const page of pageKeys) {
+        const includeGlobs = pageTraceIncludes.get(page)
+        const excludeGlobs = pageTraceExcludes.get(page)
+
+        if (!includeGlobs?.length && !excludeGlobs?.length) {
+          continue
+        }
+
+        const traceFile = path.join(
+          distDir,
+          'server/pages',
+          `${page}.js.nft.json`
+        )
+        const traceContent = JSON.parse(
+          await promises.readFile(traceFile, 'utf8')
+        )
+        let includes: string[] = []
+        let excludes: string[] = []
+
+        if (includeGlobs?.length) {
+          for (const includeGlob of includeGlobs) {
+            includes.push(...(await glob(includeGlob)))
+          }
+        }
+
+        if (excludeGlobs?.length) {
+          for (const excludeGlob of excludeGlobs) {
+            excludes.push(...(await glob(excludeGlob)))
+          }
+        }
+
+        const combined = new Set([...traceContent.files, ...includes])
+        excludes.forEach((file) => combined.delete(file))
+
+        await promises.writeFile(
+          traceFile,
+          JSON.stringify({
+            version: traceContent.version,
+            files: [...combined],
+          })
+        )
+      }
+    }
+
     if (serverPropsPages.size > 0 || ssgPages.size > 0) {
       // We update the routes manifest after the build with the
       // data routes since we can't determine these until after build

packages/next/build/utils.ts

@@ -23,7 +23,7 @@ import { getRouteMatcher, getRouteRegex } from '../shared/lib/router/utils'
 import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
 import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters'
 import { findPageFile } from '../server/lib/find-page-file'
-import { GetStaticPaths } from 'next/types'
+import { GetStaticPaths, PageConfig } from 'next/types'
 import { denormalizePagePath } from '../server/normalize-page-path'
 import { BuildManifest } from '../server/get-page-files'
 import { removePathTrailingSlash } from '../client/normalize-trailing-slash'
@@ -831,6 +831,8 @@ export async function isPageStatic(
   encodedPrerenderRoutes?: string[]
   prerenderFallback?: boolean | 'blocking'
   isNextImageImported?: boolean
+  traceIncludes?: string[]
+  traceExcludes?: string[]
 }> {
   const isPageStaticSpan = trace('is-page-static-utils', parentId)
   return isPageStaticSpan.traceAsyncFn(async () => {
@@ -925,7 +927,7 @@ export async function isPageStatic(
       }
 
       const isNextImageImported = (global as any).__NEXT_IMAGE_IMPORTED
-      const config = mod.config || {}
+      const config: PageConfig = mod.config || {}
       return {
         isStatic: !hasStaticProps && !hasGetInitialProps && !hasServerProps,
         isHybridAmp: config.amp === 'hybrid',
@@ -936,6 +938,8 @@ export async function isPageStatic(
         hasStaticProps,
         hasServerProps,
         isNextImageImported,
+        traceIncludes: config.unstable_includeFiles || [],
+        traceExcludes: config.unstable_excludeFiles || [],
       }
     } catch (err) {
       if (err.code === 'MODULE_NOT_FOUND') return {}

packages/next/build/webpack-config.ts

@@ -38,6 +38,7 @@ import BuildStatsPlugin from './webpack/plugins/build-stats-plugin'
 import ChunkNamesPlugin from './webpack/plugins/chunk-names-plugin'
 import { JsConfigPathsPlugin } from './webpack/plugins/jsconfig-paths-plugin'
 import { DropClientPage } from './webpack/plugins/next-drop-client-page-plugin'
+import { TraceEntryPointsPlugin } from './webpack/plugins/next-trace-entrypoints-plugin'
 import NextJsSsrImportPlugin from './webpack/plugins/nextjs-ssr-import'
 import NextJsSSRModuleCachePlugin from './webpack/plugins/nextjs-ssr-module-cache'
 import PagesManifestPlugin from './webpack/plugins/pages-manifest-plugin'
@@ -1245,6 +1246,12 @@ export default async function getBaseWebpackConfig(
           pagesDir,
         }),
       !isServer && new DropClientPage(),
+      config.experimental.nftTracing &&
+        !isLikeServerless &&
+        isServer &&
+        !dev &&
+        isWebpack5 &&
+        new TraceEntryPointsPlugin({ appDir: dir }),
       // Moment.js is an extremely popular library that bundles large locale files
       // by default due to how Webpack interprets its code. This is a practical
       // solution that requires the user to opt into importing specific locales.