feat: added support for prefetch segments when deployed (#75202)

Designed to enable the client prefetch segments feature when deployed on Vercel, this adds additional fields and data to the manifests to allow the CLI to create the rewrite rules needed to serve the prefetch segments. The fallback routes also now emit their RSC data to disk even though only their filenames are used. We emit the routes for the prefetches using backreferences. For example, for the route: ``` /blog/[...slug].segments/$c$slug$[...slug]/__PAGE__.segment.rsc ``` We create the following Regular Expression: ```ts /^\/blog\/(?<nxtPslug>.+?)\.segments\/\$c\$slug\$\k<nxtPslug>\/__PAGE__\.segment\.rsc$/ ``` Which enforces that the named capture group `nxtPslug` is the same for both the `[...slug].segments` and the `$c$slug$[...slug]`: ``` Matching: /blog/my-post.segments/$c$slug$my-post/__PAGE__.segment.rsc /blog/hello-world.segments/$c$slug$hello-world/__PAGE__.segment.rsc /blog/2024-01.segments/$c$slug$2024-01/__PAGE__.segment.rsc Not Matching /blog/post-1.segments/$c$slug$post-2/__PAGE__.segment.rsc ``` This also fixed a bug where the client segment prefetch keys included double `//`.  --------- Co-authored-by: JJ Kasper <jj@jjsweb.site>
vercel · Feb 4, 2025 · 06eb067 · 06eb067
1 parent fbe971c
commit 06eb067
Show file tree

Hide file tree

Showing 19 changed files with 346 additions and 48 deletions.
diff --git a/packages/next/errors.json b/packages/next/errors.json
@@ -630,5 +630,6 @@
   "629": "Invariant (SlowModuleDetectionPlugin): Unable to find the start time for a module build. This is a Next.js internal bug.",
   "630": "Invariant (SlowModuleDetectionPlugin): Module is recorded after the report is generated. This is a Next.js internal bug.",
   "631": "Invariant (SlowModuleDetectionPlugin): Circular dependency detected in module graph. This is a Next.js internal bug.",
-  "632": "Invariant (SlowModuleDetectionPlugin): Module is missing a required debugId. This is a Next.js internal bug."
+  "632": "Invariant (SlowModuleDetectionPlugin): Module is missing a required debugId. This is a Next.js internal bug.",
+  "633": "Dynamic route not found"
 }
diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts
@@ -33,6 +33,8 @@ import {
   NEXT_CACHE_REVALIDATE_TAG_TOKEN_HEADER,
   NEXT_CACHE_REVALIDATED_TAGS_HEADER,
   MATCHED_PATH_HEADER,
+  RSC_SEGMENTS_DIR_SUFFIX,
+  RSC_SEGMENT_SUFFIX,
 } from '../lib/constants'
 import { FileType, fileExists } from '../lib/file-exists'
 import { findPagesDir } from '../lib/find-pages-dir'
@@ -198,6 +200,10 @@ import {
 import { InvariantError } from '../shared/lib/invariant-error'
 import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot'
 import type { UseCacheTrackerKey } from './webpack/plugins/telemetry-plugin/use-cache-tracker-utils'
+import {
+  buildPrefetchSegmentDataRoute,
+  type PrefetchSegmentDataRoute,
+} from '../server/lib/router-utils/build-prefetch-segment-data-route'
 
 import { turbopackBuild } from './turbopack-build'
 
@@ -343,9 +349,10 @@ export type ManifestRoute = ManifestBuiltRoute & {
   page: string
   namedRegex?: string
   routeKeys?: { [key: string]: string }
+  prefetchSegmentDataRoutes?: PrefetchSegmentDataRoute[]
 }
 
-export type ManifestDataRoute = {
+type ManifestDataRoute = {
   page: string
   routeKeys?: { [key: string]: string }
   dataRouteRegex: string
@@ -387,6 +394,9 @@ export type RoutesManifest = {
     prefetchHeader: typeof NEXT_ROUTER_PREFETCH_HEADER
     suffix: typeof RSC_SUFFIX
     prefetchSuffix: typeof RSC_PREFETCH_SUFFIX
+    prefetchSegmentHeader: typeof NEXT_ROUTER_SEGMENT_PREFETCH_HEADER
+    prefetchSegmentDirSuffix: typeof RSC_SEGMENTS_DIR_SUFFIX
+    prefetchSegmentSuffix: typeof RSC_SEGMENT_SUFFIX
   }
   rewriteHeaders: {
     pathHeader: typeof NEXT_REWRITTEN_PATH_HEADER
@@ -1256,6 +1266,9 @@ export default async function build(
               contentTypeHeader: RSC_CONTENT_TYPE_HEADER,
               suffix: RSC_SUFFIX,
               prefetchSuffix: RSC_PREFETCH_SUFFIX,
+              prefetchSegmentHeader: NEXT_ROUTER_SEGMENT_PREFETCH_HEADER,
+              prefetchSegmentSuffix: RSC_SEGMENT_SUFFIX,
+              prefetchSegmentDirSuffix: RSC_SEGMENTS_DIR_SUFFIX,
             },
             rewriteHeaders: {
               pathHeader: NEXT_REWRITTEN_PATH_HEADER,
@@ -2364,8 +2377,6 @@ export default async function build(
         ]).map((page) => {
           return buildDataRoute(page, buildId)
         })
-
-        // await writeManifest(routesManifestPath, routesManifest)
       }
 
       // We need to write the manifest with rewrites before build
@@ -2926,6 +2937,24 @@ export default async function build(
                   )
                 }
 
+                if (!isAppRouteHandler && metadata?.segmentPaths) {
+                  const dynamicRoute = routesManifest.dynamicRoutes.find(
+                    (r) => r.page === page
+                  )
+                  if (!dynamicRoute) {
+                    throw new Error('Dynamic route not found')
+                  }
+
+                  dynamicRoute.prefetchSegmentDataRoutes = []
+                  for (const segmentPath of metadata.segmentPaths) {
+                    const result = buildPrefetchSegmentDataRoute(
+                      route.pathname,
+                      segmentPath
+                    )
+                    dynamicRoute.prefetchSegmentDataRoutes.push(result)
+                  }
+                }
+
                 pageInfos.set(route.pathname, {
                   ...(pageInfos.get(route.pathname) as PageInfo),
                   isDynamicAppRoute: true,
@@ -3334,6 +3363,12 @@ export default async function build(
           await fs.rm(outdir, { recursive: true, force: true })
           await writeManifest(pagesManifestPath, pagesManifest)
         })
+
+        // We need to write the manifest with rewrites after build as it might
+        // have been modified.
+        await nextBuildSpan
+          .traceChild('write-routes-manifest')
+          .traceAsyncFn(() => writeManifest(routesManifestPath, routesManifest))
       }
 
       const postBuildSpinner = createSpinner('Finalizing page optimization')

diff --git a/packages/next/src/client/components/segment-cache/cache.ts b/packages/next/src/client/components/segment-cache/cache.ts
@@ -961,7 +961,7 @@ export async function fetchSegmentOnCacheMiss(
           // It just needs to match the equivalent logic that happens when
           // prerendering the responses. It should not leak outside of Next.js.
           '/_index'
-        : '/' + segmentKeyPath,
+        : segmentKeyPath,
       routeKey.nextUrl
     )
     if (

diff --git a/packages/next/src/export/types.ts b/packages/next/src/export/types.ts
@@ -11,6 +11,7 @@ import type {
   TurborepoAccessTraceResult,
 } from '../build/turborepo-access-trace'
 import type { FetchMetrics } from '../server/base-http'
+import type { RouteMetadata } from './routes/types'
 
 export interface AmpValidation {
   page: string
@@ -67,10 +68,7 @@ export type ExportRouteResult =
   | {
       ampValidations?: AmpValidation[]
       revalidate: Revalidate
-      metadata?: {
-        status?: number
-        headers?: OutgoingHttpHeaders
-      }
+      metadata?: Partial<RouteMetadata>
       ssgNotFound?: boolean
       hasEmptyPrelude?: boolean
       hasPostponed?: boolean
@@ -135,7 +133,7 @@ export type ExportAppResult = {
       /**
        * The metadata for the page.
        */
-      metadata?: { status?: number; headers?: OutgoingHttpHeaders }
+      metadata?: Partial<RouteMetadata>
       /**
        * If the page has an empty prelude when using PPR.
        */

diff --git a/packages/next/src/lib/create-client-router-filter.ts b/packages/next/src/lib/create-client-router-filter.ts
@@ -31,7 +31,7 @@ export function createClientRouterFilter(
 
       // start at 1 since we split on '/' and the path starts
       // with this so the first entry is an empty string
-      for (let i = 1; i < pathParts.length + 1; i++) {
+      for (let i = 1; i < pathParts.length; i++) {
         const curPart = pathParts[i]
 
         if (curPart.startsWith('[')) {

diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx
@@ -2488,13 +2488,9 @@ type PrerenderToStreamResult = {
  * Determines whether we should generate static flight data.
  */
 function shouldGenerateStaticFlightData(workStore: WorkStore): boolean {
-  const { fallbackRouteParams, isStaticGeneration } = workStore
+  const { isStaticGeneration } = workStore
   if (!isStaticGeneration) return false
 
-  if (fallbackRouteParams && fallbackRouteParams.size > 0) {
-    return false
-  }
-
   return true
 }
 

diff --git a/packages/next/src/server/app-render/collect-segment-data.tsx b/packages/next/src/server/app-render/collect-segment-data.tsx
@@ -379,7 +379,7 @@ async function renderSegmentPrefetch(
   if (key === ROOT_SEGMENT_KEY) {
     return ['/_index', segmentBuffer]
   } else {
-    return ['/' + key, segmentBuffer]
+    return [key, segmentBuffer]
   }
 }
 

diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts
@@ -176,6 +176,7 @@ import { FallbackMode, parseFallbackField } from '../lib/fallback'
 import { toResponseCacheEntry } from './response-cache/utils'
 import { scheduleOnNextTick } from '../lib/scheduler'
 import { shouldServeStreamingMetadata } from './lib/streaming-metadata'
+import { SegmentPrefixRSCPathnameNormalizer } from './normalizers/request/segment-prefix-rsc'
 
 export type FindComponentsResult = {
   components: LoadComponentsReturnType
@@ -450,10 +451,12 @@ export default abstract class Server<
   protected readonly normalizers: {
     readonly rsc: RSCPathnameNormalizer | undefined
     readonly prefetchRSC: PrefetchRSCPathnameNormalizer | undefined
+    readonly segmentPrefetchRSC: SegmentPrefixRSCPathnameNormalizer | undefined
     readonly data: NextDataPathnameNormalizer | undefined
   }
 
   private readonly isAppPPREnabled: boolean
+  private readonly isAppSegmentPrefetchEnabled: boolean
 
   /**
    * This is used to persist cache scopes across
@@ -529,6 +532,10 @@ export default abstract class Server<
       this.enabledDirectories.app &&
       checkIsAppPPREnabled(this.nextConfig.experimental.ppr)
 
+    this.isAppSegmentPrefetchEnabled =
+      this.enabledDirectories.app &&
+      this.nextConfig.experimental.clientSegmentCache === true
+
     this.normalizers = {
       // We should normalize the pathname from the RSC prefix only in minimal
       // mode as otherwise that route is not exposed external to the server as
@@ -541,6 +548,10 @@ export default abstract class Server<
         this.isAppPPREnabled && this.minimalMode
           ? new PrefetchRSCPathnameNormalizer()
           : undefined,
+      segmentPrefetchRSC:
+        this.isAppSegmentPrefetchEnabled && this.minimalMode
+          ? new SegmentPrefixRSCPathnameNormalizer()
+          : undefined,
       data: this.enabledDirectories.pages
         ? new NextDataPathnameNormalizer(this.buildId)
         : undefined,
@@ -637,7 +648,25 @@ export default abstract class Server<
   ) => {
     if (!parsedUrl.pathname) return false
 
-    if (this.normalizers.prefetchRSC?.match(parsedUrl.pathname)) {
+    if (this.normalizers.segmentPrefetchRSC?.match(parsedUrl.pathname)) {
+      const result = this.normalizers.segmentPrefetchRSC.extract(
+        parsedUrl.pathname
+      )
+      if (!result) return false
+
+      const { originalPathname, segmentPath } = result
+      parsedUrl.pathname = originalPathname
+
+      // Mark the request as a router prefetch request.
+      req.headers[RSC_HEADER.toLowerCase()] = '1'
+      req.headers[NEXT_ROUTER_PREFETCH_HEADER.toLowerCase()] = '1'
+      req.headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER.toLowerCase()] =
+        segmentPath
+
+      addRequestMeta(req, 'isRSCRequest', true)
+      addRequestMeta(req, 'isPrefetchRSCRequest', true)
+      addRequestMeta(req, 'segmentPrefetchRSCRequest', segmentPath)
+    } else if (this.normalizers.prefetchRSC?.match(parsedUrl.pathname)) {
       parsedUrl.pathname = this.normalizers.prefetchRSC.normalize(
         parsedUrl.pathname,
         true
@@ -671,6 +700,16 @@ export default abstract class Server<
 
       if (req.headers[NEXT_ROUTER_PREFETCH_HEADER.toLowerCase()] === '1') {
         addRequestMeta(req, 'isPrefetchRSCRequest', true)
+
+        const segmentPrefetchRSCRequest =
+          req.headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER.toLowerCase()]
+        if (typeof segmentPrefetchRSCRequest === 'string') {
+          addRequestMeta(
+            req,
+            'segmentPrefetchRSCRequest',
+            segmentPrefetchRSCRequest
+          )
+        }
       }
     } else {
       // Otherwise just return without doing anything.
@@ -1288,6 +1327,31 @@ export default abstract class Server<
             if (params) {
               matchedPath = utils.interpolateDynamicPath(srcPathname, params)
               req.url = utils.interpolateDynamicPath(req.url!, params)
+
+              // If the request is for a segment prefetch, we need to update the
+              // segment prefetch request path to include the interpolated
+              // params.
+              let segmentPrefetchRSCRequest = getRequestMeta(
+                req,
+                'segmentPrefetchRSCRequest'
+              )
+              if (
+                segmentPrefetchRSCRequest &&
+                isDynamicRoute(segmentPrefetchRSCRequest, false)
+              ) {
+                segmentPrefetchRSCRequest = utils.interpolateDynamicPath(
+                  segmentPrefetchRSCRequest,
+                  params
+                )
+
+                req.headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER.toLowerCase()] =
+                  segmentPrefetchRSCRequest
+                addRequestMeta(
+                  req,
+                  'segmentPrefetchRSCRequest',
+                  segmentPrefetchRSCRequest
+                )
+              }
             }
           }
 
@@ -1541,6 +1605,12 @@ export default abstract class Server<
       normalizers.push(this.normalizers.data)
     }
 
+    // We have to put the segment prefetch normalizer before the RSC normalizer
+    // because the RSC normalizer will match the prefetch RSC routes too.
+    if (this.normalizers.segmentPrefetchRSC) {
+      normalizers.push(this.normalizers.segmentPrefetchRSC)
+    }
+
     // We have to put the prefetch normalizer before the RSC normalizer
     // because the RSC normalizer will match the prefetch RSC routes too.
     if (this.normalizers.prefetchRSC) {
@@ -2139,8 +2209,10 @@ export default abstract class Server<
     // need to transfer it to the request meta because it's only read
     // within this function; the static segment data should have already been
     // generated, so we will always either return a static response or a 404.
-    const segmentPrefetchHeader =
-      req.headers[NEXT_ROUTER_SEGMENT_PREFETCH_HEADER.toLowerCase()]
+    const segmentPrefetchHeader = getRequestMeta(
+      req,
+      'segmentPrefetchRSCRequest'
+    )
 
     // we need to ensure the status code if /404 is visited directly
     if (is404Page && !isNextDataRequest && !isRSCRequest) {
@@ -3170,7 +3242,11 @@ export default abstract class Server<
 
     const { value: cachedData } = cacheEntry
 
-    if (isRoutePPREnabled && typeof segmentPrefetchHeader === 'string') {
+    if (
+      typeof segmentPrefetchHeader === 'string' &&
+      cachedData?.kind === CachedRouteKind.APP_PAGE &&
+      cachedData.segmentData
+    ) {
       // This is a prefetch request issued by the client Segment Cache. These
       // should never reach the application layer (lambda). We should either
       // respond from the cache (HIT) or respond with 204 No Content (MISS).
@@ -3183,23 +3259,15 @@ export default abstract class Server<
       // response itself contains whether the data is dynamic.
       res.setHeader(NEXT_DID_POSTPONE_HEADER, '2')
 
-      if (
-        // This is always true at runtime but is needed to refine the type
-        // of cacheEntry.value to CachedAppPageValue, because the outer
-        // ResponseCacheEntry is not a discriminated union.
-        cachedData?.kind === CachedRouteKind.APP_PAGE &&
-        cachedData.segmentData
-      ) {
-        const matchedSegment = cachedData.segmentData.get(segmentPrefetchHeader)
-        if (matchedSegment !== undefined) {
-          // Cache hit
-          return {
-            type: 'rsc',
-            body: RenderResult.fromStatic(matchedSegment),
-            // TODO: Eventually this should use revalidate time of the
-            // individual segment, not the whole page.
-            revalidate: cacheEntry.revalidate,
-          }
+      const matchedSegment = cachedData.segmentData.get(segmentPrefetchHeader)
+      if (matchedSegment !== undefined) {
+        // Cache hit
+        return {
+          type: 'rsc',
+          body: RenderResult.fromStatic(matchedSegment),
+          // TODO: Eventually this should use revalidate time of the
+          // individual segment, not the whole page.
+          revalidate: cacheEntry.revalidate,
         }
       }
 

diff --git a/packages/next/src/server/lib/router-utils/build-prefetch-segment-data-route.test.ts b/packages/next/src/server/lib/router-utils/build-prefetch-segment-data-route.test.ts
@@ -0,0 +1,17 @@
+import { buildPrefetchSegmentDataRoute } from './build-prefetch-segment-data-route'
+
+describe('buildPrefetchSegmentDataRoute', () => {
+  it('should build a prefetch segment data route', () => {
+    const route = buildPrefetchSegmentDataRoute(
+      '/blog/[...slug]',
+      '/$c$slug$[slug]/__PAGE__'
+    )
+
+    expect(route).toMatchInlineSnapshot(`
+     {
+       "destination": "/blog/[...slug].segments/$c$slug$[slug]/__PAGE__.segment.rsc",
+       "source": "^/blog/(?<nxtPslug>.+?)\\.segments/\\$c\\$slug\\$\\k<nxtPslug>/__PAGE__\\.segment\\.rsc$",
+     }
+    `)
+  })
+})