Skip to content

Commit 3387ac9

Browse files
james-elicxdario-piotrowicz
james-elicx
and
dario-piotrowicz
authored
feat: new routing system build time changes (#153)
* feat: utility functions for route path names * feat: format path names + check invalid generated functions after * feat: adjust middleware manifest to support new route system fixes * feat: process vercel config and build output * feat: connect new build time to worker script * chore: changeset * chore: add issue reference to comments * feat: `VercelBuildOutputItem` type Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: refine comment Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: refine comment Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: apply suggestions from code review Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: address suggestions * chore: make prettier happy * test(fs): readPathsRecursively * test: routing utils * fix: invalid rsc functions with valid squashed non-rsc functions * chore: move part of comment * feat: use `relative` for static asset name resolution * feat: add build output config overrides to build output map * chore: change type to be of override instead of static * fix: wrong type in test * fix: derp, fix type error * chore: apply suggestions from code review Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: prettier * fix: add back `/page` checking for older nextjs versions middleware manifests * chore: apply suggestions from code review Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> * chore: prettier * feat: improve vercel types once more * Update src/buildApplication/middlewareManifest.ts * chore: apply suggestions from code review Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com> --------- Co-authored-by: Dario Piotrowicz <dario.piotrowicz@gmail.com>
1 parent 9834066 commit 3387ac9

23 files changed

+1025
-144
lines changed

‎.changeset/spotty-cows-add.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@cloudflare/next-on-pages': minor
3+
---
4+
5+
New routing system build time processing + integration with worker script.

‎src/buildApplication/buildApplication.ts

+19-6
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ import {
1515
} from './buildVercelOutput';
1616
import { buildMetadataFiles } from './buildMetadataFiles';
1717
import { validateDir } from '../utils';
18+
import {
19+
getVercelStaticAssets,
20+
processVercelOutput,
21+
} from './processVercelOutput';
1822

1923
/**
2024
* Builds the _worker.js with static assets implementing the Next.js application
@@ -62,9 +66,7 @@ async function prepareAndBuildWorker(
6266
cliLog(`Using basePath ${nextJsConfigs.basePath}`);
6367
}
6468

65-
const functionsDir = resolve(
66-
`.vercel/output/functions${nextJsConfigs.basePath ?? ''}`
67-
);
69+
const functionsDir = resolve(`.vercel/output/functions`);
6870
if (!(await validateDir(functionsDir))) {
6971
cliLog('No functions detected.');
7072
return;
@@ -80,10 +82,14 @@ async function prepareAndBuildWorker(
8082
return;
8183
}
8284

85+
// NOTE: Middleware manifest logic will be removed in the new routing system. (see issue #129)
8386
let middlewareManifestData: MiddlewareManifestData;
8487

8588
try {
86-
middlewareManifestData = await getParsedMiddlewareManifest(functionsMap);
89+
middlewareManifestData = await getParsedMiddlewareManifest(
90+
functionsMap,
91+
nextJsConfigs
92+
);
8793
} catch (e: unknown) {
8894
if (e instanceof Error) {
8995
cliError(e.message, true);
@@ -96,9 +102,16 @@ async function prepareAndBuildWorker(
96102
exit(1);
97103
}
98104

99-
await buildWorkerFile(
100-
middlewareManifestData,
105+
const staticAssets = await getVercelStaticAssets();
106+
107+
const processedVercelOutput = processVercelOutput(
101108
vercelConfig,
109+
staticAssets,
110+
middlewareManifestData
111+
);
112+
113+
await buildWorkerFile(
114+
processedVercelOutput,
102115
nextJsConfigs,
103116
options.experimentalMinify
104117
);

‎src/buildApplication/buildWorkerFile.ts

+27-19
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,34 @@ import { build } from 'esbuild';
44
import { tmpdir } from 'os';
55
import { cliLog } from '../cli';
66
import { NextJsConfigs } from './nextJsConfigs';
7-
import { MiddlewareManifestData } from './middlewareManifest';
87
import { generateGlobalJs } from './generateGlobalJs';
8+
import { ProcessedVercelOutput } from './processVercelOutput';
99

10+
/**
11+
* Construct a record for the build output map.
12+
*
13+
* @param item The build output item to construct a record for.
14+
* @returns Record for the build output map.
15+
*/
16+
function constructBuildOutputRecord(item: BuildOutputItem) {
17+
return item.type === 'static'
18+
? `{ type: ${JSON.stringify(item.type)} }`
19+
: item.type === 'override'
20+
? `{
21+
type: ${JSON.stringify(item.type)},
22+
path: ${item.path ? JSON.stringify(item.path) : undefined},
23+
contentType: ${item.contentType ? JSON.stringify(item.contentType) : undefined}
24+
}`
25+
: `{
26+
type: ${JSON.stringify(item.type)},
27+
entrypoint: AsyncLocalStoragePromise.then(() => import('${item.entrypoint}')),
28+
matchers: ${JSON.stringify(item.matchers)}
29+
}`;
30+
}
31+
32+
// NOTE: `nextJsConfigs`, and accompanying logic will be removed in the new routing system. (see issue #129)
1033
export async function buildWorkerFile(
11-
{ hydratedMiddleware, hydratedFunctions }: MiddlewareManifestData,
12-
vercelConfig: VercelConfig,
34+
{ vercelConfig, vercelOutput }: ProcessedVercelOutput,
1335
nextJsConfigs: NextJsConfigs,
1436
experimentalMinify: boolean
1537
) {
@@ -25,22 +47,8 @@ export async function buildWorkerFile(
2547
globalThis.AsyncLocalStorage = AsyncLocalStorage;
2648
}).catch(() => undefined);
2749
28-
export const __FUNCTIONS__ = {${[...hydratedFunctions.entries()]
29-
.map(
30-
([name, { matchers, filepath }]) =>
31-
`"${name}": { matchers: ${JSON.stringify(
32-
matchers
33-
)}, entrypoint: AsyncLocalStoragePromise.then(() => import('${filepath}'))}`
34-
)
35-
.join(',')}};
36-
37-
export const __MIDDLEWARE__ = {${[...hydratedMiddleware.entries()]
38-
.map(
39-
([name, { matchers, filepath }]) =>
40-
`"${name}": { matchers: ${JSON.stringify(
41-
matchers
42-
)}, entrypoint: AsyncLocalStoragePromise.then(() => import('${filepath}'))}`
43-
)
50+
export const __BUILD_OUTPUT__ = {${[...vercelOutput.entries()]
51+
.map(([name, item]) => `"${name}": ${constructBuildOutputRecord(item)}`)
4452
.join(',')}};`
4553
);
4654

‎src/buildApplication/generateFunctionsMap.ts

+82-11
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ import { dirname, join, relative } from 'path';
44
import { parse, Node } from 'acorn';
55
import { generate } from 'astring';
66
import {
7+
formatRoutePath,
78
normalizePath,
89
readJsonFile,
10+
stripIndexRoute,
911
validateDir,
1012
validateFile,
1113
} from '../utils';
12-
import { cliError, CliOptions } from '../cli';
14+
import { cliError, CliOptions, cliWarn } from '../cli';
1315
import { tmpdir } from 'os';
1416

1517
/**
@@ -27,10 +29,9 @@ import { tmpdir } from 'os';
2729
export async function generateFunctionsMap(
2830
functionsDir: string,
2931
experimentalMinify: CliOptions['experimentalMinify']
30-
): Promise<{
31-
functionsMap: Map<string, string>;
32-
invalidFunctions: Set<string>;
33-
}> {
32+
): Promise<
33+
Pick<DirectoryProcessingResults, 'functionsMap' | 'invalidFunctions'>
34+
> {
3435
const processingSetup = {
3536
functionsDir,
3637
tmpFunctionsDir: join(tmpdir(), Math.random().toString(36).slice(2)),
@@ -43,6 +44,8 @@ export async function generateFunctionsMap(
4344
functionsDir
4445
);
4546

47+
await tryToFixInvalidFunctions(processingResults);
48+
4649
if (experimentalMinify) {
4750
await buildWebpackChunkFiles(
4851
processingResults.webpackChunks,
@@ -53,6 +56,53 @@ export async function generateFunctionsMap(
5356
return processingResults;
5457
}
5558

59+
/**
60+
* Process the invalid functions and check whether and valid function was created in the functions
61+
* map to override it.
62+
*
63+
* The build output sometimes generates invalid functions at the root, while still creating the
64+
* valid functions. With the base path and route groups, it might create the valid edge function
65+
* inside a folder for the route group, but create an invalid one that maps to the same path
66+
* at the root.
67+
*
68+
* When we process the directory, we might add the valid function to the map before we process the
69+
* invalid one, so we need to check if the invalid one was added to the map and remove it from the
70+
* set if it was.
71+
*
72+
* If the invalid function is an RSC function (e.g. `path.rsc`) and doesn't have a valid squashed
73+
* version, we check if a squashed non-RSC function exists (e.g. `path`) and use this instead. RSC
74+
* functions are the same as non-RSC functions, per the Vercel source code.
75+
* https://github.com/vercel/vercel/blob/main/packages/next/src/server-build.ts#L1193
76+
*
77+
* @param processingResults Object containing the results of processing the current function directory.
78+
*/
79+
async function tryToFixInvalidFunctions({
80+
functionsMap,
81+
invalidFunctions,
82+
}: DirectoryProcessingResults): Promise<void> {
83+
if (invalidFunctions.size === 0) {
84+
return;
85+
}
86+
87+
for (const rawPath of invalidFunctions) {
88+
const formattedPath = formatRoutePath(rawPath);
89+
90+
if (
91+
functionsMap.has(formattedPath) ||
92+
functionsMap.has(stripIndexRoute(formattedPath))
93+
) {
94+
invalidFunctions.delete(rawPath);
95+
} else if (formattedPath.endsWith('.rsc')) {
96+
const value = functionsMap.get(formattedPath.replace(/\.rsc$/, ''));
97+
98+
if (value) {
99+
functionsMap.set(formattedPath, value);
100+
invalidFunctions.delete(rawPath);
101+
}
102+
}
103+
}
104+
}
105+
56106
async function processDirectoryRecursively(
57107
setup: ProcessingSetup,
58108
dir: string
@@ -117,8 +167,26 @@ async function processFuncDirectory(
117167
};
118168
}
119169

170+
// There are instances where the build output will generate an uncompiled `middleware.js` file that is used as the entrypoint.
171+
// TODO: investigate when and where the file is generated.
172+
// This file is not able to be used as it is uncompiled, so we try to instead use the compiled `index.js` if it exists.
173+
let isMiddleware = false;
174+
if (functionConfig.entrypoint === 'middleware.js') {
175+
isMiddleware = true;
176+
functionConfig.entrypoint = 'index.js';
177+
}
178+
120179
const functionFile = join(filepath, functionConfig.entrypoint);
121180
if (!(await validateFile(functionFile))) {
181+
if (isMiddleware) {
182+
// We sometimes encounter an uncompiled `middleware.js` with no compiled `index.js` outside of a base path.
183+
// Outside the base path, it should not be utilised, so it should be safe to ignore the function.
184+
cliWarn(
185+
`Detected an invalid middleware function for ${relativePath}. Skipping...`
186+
);
187+
return {};
188+
}
189+
122190
return {
123191
invalidFunctions: new Set([file]),
124192
};
@@ -143,12 +211,15 @@ async function processFuncDirectory(
143211
await mkdir(dirname(newFilePath), { recursive: true });
144212
await writeFile(newFilePath, contents);
145213

146-
functionsMap.set(
147-
normalizePath(
148-
relative(setup.functionsDir, filepath).slice(0, -'.func'.length)
149-
),
150-
normalizePath(newFilePath)
151-
);
214+
const formattedPathName = formatRoutePath(relativePath);
215+
const normalizedFilePath = normalizePath(newFilePath);
216+
217+
functionsMap.set(formattedPathName, normalizedFilePath);
218+
219+
if (formattedPathName.endsWith('/index')) {
220+
// strip `/index` from the path name as the build output config doesn't rewrite `/index` to `/`
221+
functionsMap.set(stripIndexRoute(formattedPathName), normalizedFilePath);
222+
}
152223

153224
return {
154225
functionsMap,

‎src/buildApplication/getVercelConfig.ts

+28
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,31 @@ export async function getVercelConfig(): Promise<VercelConfig> {
2525

2626
return config;
2727
}
28+
29+
export function processVercelConfig(
30+
config: VercelConfig
31+
): ProcessedVercelConfig {
32+
const processedConfig: ProcessedVercelConfig = {
33+
...config,
34+
routes: {
35+
none: [],
36+
filesystem: [],
37+
miss: [],
38+
rewrite: [],
39+
resource: [],
40+
hit: [],
41+
error: [],
42+
},
43+
};
44+
45+
let currentPhase: VercelHandleValue | 'none' = 'none';
46+
config.routes.forEach(route => {
47+
if ('handle' in route) {
48+
currentPhase = route.handle;
49+
} else {
50+
processedConfig.routes[currentPhase].push(route);
51+
}
52+
});
53+
54+
return processedConfig;
55+
}

‎src/buildApplication/middlewareManifest.ts

+23-6
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@
44
* should be refactored to use .vercel/output instead as soon as possible
55
*/
66

7-
import { readJsonFile } from '../utils';
7+
// NOTE: This file and the corresponding logic will be removed in the new routing system. (see issue #129)
8+
9+
import { readJsonFile, stripIndexRoute, stripRouteGroups } from '../utils';
10+
import type { NextJsConfigs } from './nextJsConfigs';
811

912
export type EdgeFunctionDefinition = {
1013
name: string;
@@ -31,7 +34,8 @@ export type MiddlewareManifestData = Awaited<
3134
* gets the parsed middleware manifest and validates it against the existing functions map.
3235
*/
3336
export async function getParsedMiddlewareManifest(
34-
functionsMap: Map<string, string>
37+
functionsMap: Map<string, string>,
38+
{ basePath }: NextJsConfigs
3539
) {
3640
// Annoying that we don't get this from the `.vercel` directory.
3741
// Maybe we eventually just construct something similar from the `.vercel/output/functions` directory with the same magic filename/precendence rules?
@@ -42,12 +46,13 @@ export async function getParsedMiddlewareManifest(
4246
throw new Error('Could not read the functions manifest.');
4347
}
4448

45-
return parseMiddlewareManifest(middlewareManifest, functionsMap);
49+
return parseMiddlewareManifest(middlewareManifest, functionsMap, basePath);
4650
}
4751

4852
export function parseMiddlewareManifest(
4953
middlewareManifest: MiddlewareManifest,
50-
functionsMap: Map<string, string>
54+
functionsMap: Map<string, string>,
55+
basePath?: string
5156
) {
5257
if (middlewareManifest.version !== 2) {
5358
throw new Error(
@@ -74,9 +79,13 @@ export function parseMiddlewareManifest(
7479
const functionsEntries = Object.values(middlewareManifest.functions);
7580

7681
for (const [name, filepath] of functionsMap) {
82+
// the .vc-config name includes the basePath, so we need to strip it for matching in the middleware manifest.
83+
const prefixRegex = new RegExp(`^(${basePath})?/`);
84+
const fileName = name.replace(prefixRegex, '');
85+
7786
if (
7887
middlewareEntries.length > 0 &&
79-
(name === 'middleware' || name === 'src/middleware')
88+
(fileName === 'middleware' || fileName === 'src/middleware')
8089
) {
8190
for (const entry of middlewareEntries) {
8291
if (entry?.name === 'middleware' || entry?.name === 'src/middleware') {
@@ -86,8 +95,16 @@ export function parseMiddlewareManifest(
8695
}
8796

8897
for (const entry of functionsEntries) {
89-
if (matchFunctionEntry(entry.name, name)) {
98+
if (matchFunctionEntry(stripRouteGroups(entry.name), fileName)) {
9099
hydratedFunctions.set(name, { matchers: entry.matchers, filepath });
100+
101+
// NOTE: Temporary to account for `/index` routes.
102+
if (stripIndexRoute(name) !== name) {
103+
hydratedFunctions.set(stripIndexRoute(name), {
104+
matchers: entry.matchers,
105+
filepath,
106+
});
107+
}
91108
}
92109
}
93110
}

‎src/buildApplication/nextJsConfigs/getBasePath.ts

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// NOTE: This file and the corresponding logic will be removed in the new routing system. (see issue #129)
2+
13
import { cliWarn } from '../../cli';
24
import { readJsonFile } from '../../utils';
35

‎src/buildApplication/nextJsConfigs/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// NOTE: This file and the corresponding logic will be removed in the new routing system. (see issue #129)
2+
13
import { getBasePath } from './getBasePath';
24

35
export type NextJsConfigs = {

0 commit comments

Comments
 (0)