Skip to content

Commit 3692a5e

Browse files
ijjkijjk
authored
Add falling back to wasm swc build on native load failure (#36612)
Follow-up to #36527 this adds falling back to the wasm swc build when loading the native bindings fails so that we don't block the build on the native dependency being available. This continues off of #33496 but does not add a postinstall script yet and only downloads the fallback when the native dependency fails to load.
1 parent fcec758 commit 3692a5e

File tree

12 files changed

+309
-82
lines changed

12 files changed

+309
-82
lines changed

‎packages/next/build/swc/index.js‎

Lines changed: 78 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,81 @@
1+
import path from 'path'
2+
import { pathToFileURL } from 'url'
13
import { platform, arch } from 'os'
24
import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples'
3-
import { version as nextVersion, optionalDependencies } from 'next/package.json'
45
import * as Log from '../output/log'
56
import { getParserOptions } from './options'
67
import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
78
import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
9+
import { downloadWasmSwc } from '../../lib/download-wasm-swc'
10+
import { version as nextVersion } from 'next/package.json'
811

912
const ArchName = arch()
1013
const PlatformName = platform()
1114
const triples = platformArchTriples[PlatformName][ArchName] || []
1215

1316
let nativeBindings
1417
let wasmBindings
18+
let downloadWasmPromise
19+
let pendingBindings
1520
export const lockfilePatchPromise = {}
1621

1722
async function loadBindings() {
18-
if (!lockfilePatchPromise.cur) {
19-
// always run lockfile check once so that it gets patched
20-
// even if it doesn't fail to load locally
21-
lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
22-
console.error
23-
)
23+
if (pendingBindings) {
24+
return pendingBindings
2425
}
26+
pendingBindings = new Promise(async (resolve, reject) => {
27+
if (!lockfilePatchPromise.cur) {
28+
// always run lockfile check once so that it gets patched
29+
// even if it doesn't fail to load locally
30+
lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
31+
console.error
32+
)
33+
}
2534

26-
let attempts = []
27-
try {
28-
return loadNative()
29-
} catch (a) {
30-
attempts = attempts.concat(a)
31-
}
35+
let attempts = []
36+
try {
37+
return resolve(loadNative())
38+
} catch (a) {
39+
attempts = attempts.concat(a)
40+
}
3241

33-
// TODO: fetch wasm and fallback when loading native fails
34-
// so that users aren't blocked on this, we still want to
35-
// report the native load failure so we can patch though
36-
try {
37-
let bindings = await loadWasm()
38-
return bindings
39-
} catch (a) {
40-
attempts = attempts.concat(a)
41-
}
42+
try {
43+
let bindings = await loadWasm()
44+
eventSwcLoadFailure({ wasm: 'enabled' })
45+
return resolve(bindings)
46+
} catch (a) {
47+
attempts = attempts.concat(a)
48+
}
4249

43-
logLoadFailure(attempts)
50+
try {
51+
// if not installed already download wasm package on-demand
52+
// we download to a custom directory instead of to node_modules
53+
// as node_module import attempts are cached and can't be re-attempted
54+
// x-ref: https://github.com/nodejs/modules/issues/307
55+
const wasmDirectory = path.join(
56+
path.dirname(require.resolve('next/package.json')),
57+
'wasm'
58+
)
59+
if (!downloadWasmPromise) {
60+
downloadWasmPromise = downloadWasmSwc(nextVersion, wasmDirectory)
61+
}
62+
await downloadWasmPromise
63+
let bindings = await loadWasm(pathToFileURL(wasmDirectory).href)
64+
eventSwcLoadFailure({ wasm: 'fallback' })
65+
66+
// still log native load attempts so user is
67+
// aware it failed and should be fixed
68+
for (const attempt of attempts) {
69+
Log.warn(attempt)
70+
}
71+
return resolve(bindings)
72+
} catch (a) {
73+
attempts = attempts.concat(a)
74+
}
75+
76+
logLoadFailure(attempts, true)
77+
})
78+
return pendingBindings
4479
}
4580

4681
function loadBindingsSync() {
@@ -56,47 +91,16 @@ function loadBindingsSync() {
5691

5792
let loggingLoadFailure = false
5893

59-
function logLoadFailure(attempts) {
94+
function logLoadFailure(attempts, triedWasm = false) {
6095
// make sure we only emit the event and log the failure once
6196
if (loggingLoadFailure) return
6297
loggingLoadFailure = true
6398

6499
for (let attempt of attempts) {
65100
Log.warn(attempt)
66101
}
67-
let glibcVersion
68-
let installedSwcPackages
69102

70-
try {
71-
glibcVersion = process.report?.getReport().header.glibcVersionRuntime
72-
} catch (_) {}
73-
74-
try {
75-
const pkgNames = Object.keys(optionalDependencies || {}).filter((pkg) =>
76-
pkg.startsWith('@next/swc')
77-
)
78-
const installedPkgs = []
79-
80-
for (const pkg of pkgNames) {
81-
try {
82-
const { version } = require(`${pkg}/package.json`)
83-
installedPkgs.push(`${pkg}@${version}`)
84-
} catch (_) {}
85-
}
86-
87-
if (installedPkgs.length > 0) {
88-
installedSwcPackages = installedPkgs.sort().join(',')
89-
}
90-
} catch (_) {}
91-
92-
eventSwcLoadFailure({
93-
nextVersion,
94-
glibcVersion,
95-
installedSwcPackages,
96-
arch: process.arch,
97-
platform: process.platform,
98-
nodeVersion: process.versions.node,
99-
})
103+
eventSwcLoadFailure({ wasm: triedWasm ? 'failed' : undefined })
100104
.then(() => lockfilePatchPromise.cur || Promise.resolve())
101105
.finally(() => {
102106
Log.error(
@@ -106,15 +110,21 @@ function logLoadFailure(attempts) {
106110
})
107111
}
108112

109-
async function loadWasm() {
113+
async function loadWasm(importPath = '') {
110114
if (wasmBindings) {
111115
return wasmBindings
112116
}
113117

114118
let attempts = []
115119
for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
116120
try {
117-
let bindings = await import(pkg)
121+
let pkgPath = pkg
122+
123+
if (importPath) {
124+
// the import path must be exact when not in node_modules
125+
pkgPath = path.join(importPath, pkg, 'wasm.js')
126+
}
127+
let bindings = await import(pkgPath)
118128
if (pkg === '@next/swc-wasm-web') {
119129
bindings = await bindings.default()
120130
}
@@ -139,14 +149,16 @@ async function loadWasm() {
139149
}
140150
return wasmBindings
141151
} catch (e) {
142-
// Do not report attempts to load wasm when it is still experimental
143-
// if (e?.code === 'ERR_MODULE_NOT_FOUND') {
144-
// attempts.push(`Attempted to load ${pkg}, but it was not installed`)
145-
// } else {
146-
// attempts.push(
147-
// `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
148-
// )
149-
// }
152+
// Only log attempts for loading wasm when loading as fallback
153+
if (importPath) {
154+
if (e?.code === 'ERR_MODULE_NOT_FOUND') {
155+
attempts.push(`Attempted to load ${pkg}, but it was not installed`)
156+
} else {
157+
attempts.push(
158+
`Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
159+
)
160+
}
161+
}
150162
}
151163
}
152164

‎packages/next/compiled/tar/LICENSE‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
The ISC License
2+
3+
Copyright (c) Isaac Z. Schlueter and Contributors
4+
5+
Permission to use, copy, modify, and/or distribute this software for any
6+
purpose with or without fee is hereby granted, provided that the above
7+
copyright notice and this permission notice appear in all copies.
8+
9+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
15+
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

‎packages/next/compiled/tar/index.js‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/next/compiled/tar/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"name":"tar","main":"index.js","author":"Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)","license":"ISC"}

‎packages/next/lib/download-wasm-swc.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import os from 'os'
2+
import fs from 'fs'
3+
import path from 'path'
4+
import * as Log from '../build/output/log'
5+
import { execSync } from 'child_process'
6+
import tar from 'next/dist/compiled/tar'
7+
import fetch from 'next/dist/compiled/node-fetch'
8+
import { fileExists } from './file-exists'
9+
10+
const MAX_VERSIONS_TO_CACHE = 5
11+
12+
export async function downloadWasmSwc(
13+
version: string,
14+
wasmDirectory: string,
15+
variant: 'nodejs' | 'web' = 'nodejs'
16+
) {
17+
const pkgName = `@next/swc-wasm-${variant}`
18+
const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
19+
const outputDirectory = path.join(wasmDirectory, pkgName)
20+
21+
if (await fileExists(outputDirectory)) {
22+
// if the package is already downloaded a different
23+
// failure occurred than not being present
24+
return
25+
}
26+
27+
// get platform specific cache directory adapted from playwright's handling
28+
// https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
29+
const cacheDirectory = (() => {
30+
let result
31+
const envDefined = process.env['NEXT_SWC_PATH']
32+
33+
if (envDefined) {
34+
result = envDefined
35+
} else {
36+
let systemCacheDirectory
37+
if (process.platform === 'linux') {
38+
systemCacheDirectory =
39+
process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
40+
} else if (process.platform === 'darwin') {
41+
systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
42+
} else if (process.platform === 'win32') {
43+
systemCacheDirectory =
44+
process.env.LOCALAPPDATA ||
45+
path.join(os.homedir(), 'AppData', 'Local')
46+
} else {
47+
console.error(new Error('Unsupported platform: ' + process.platform))
48+
process.exit(0)
49+
}
50+
result = path.join(systemCacheDirectory, 'next-swc')
51+
}
52+
53+
if (!path.isAbsolute(result)) {
54+
// It is important to resolve to the absolute path:
55+
// - for unzipping to work correctly;
56+
// - so that registry directory matches between installation and execution.
57+
// INIT_CWD points to the root of `npm/yarn install` and is probably what
58+
// the user meant when typing the relative path.
59+
result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
60+
}
61+
return result
62+
})()
63+
64+
await fs.promises.mkdir(outputDirectory, { recursive: true })
65+
66+
const extractFromTar = async () => {
67+
await tar.x({
68+
file: path.join(cacheDirectory, tarFileName),
69+
cwd: outputDirectory,
70+
strip: 1,
71+
})
72+
}
73+
74+
if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
75+
Log.info('Downloading WASM swc package...')
76+
await fs.promises.mkdir(cacheDirectory, { recursive: true })
77+
const tempFile = path.join(
78+
cacheDirectory,
79+
`${tarFileName}.temp-${Date.now()}`
80+
)
81+
let registry = `https://registry.npmjs.org/`
82+
83+
try {
84+
const output = execSync('npm config get registry').toString().trim()
85+
if (output.startsWith('http')) {
86+
registry = output
87+
}
88+
} catch (_) {}
89+
90+
await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
91+
if (!res.ok) {
92+
throw new Error(`request failed with status ${res.status}`)
93+
}
94+
const cacheWriteStream = fs.createWriteStream(tempFile)
95+
96+
return new Promise<void>((resolve, reject) => {
97+
res.body
98+
.pipe(cacheWriteStream)
99+
.on('error', (err) => reject(err))
100+
.on('finish', () => resolve())
101+
}).finally(() => cacheWriteStream.close())
102+
})
103+
await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
104+
}
105+
await extractFromTar()
106+
107+
const cacheFiles = await fs.promises.readdir(cacheDirectory)
108+
109+
if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
110+
cacheFiles.sort()
111+
112+
for (let i = MAX_VERSIONS_TO_CACHE - 1; i++; i < cacheFiles.length) {
113+
await fs.promises
114+
.unlink(path.join(cacheDirectory, cacheFiles[i]))
115+
.catch(() => {})
116+
}
117+
}
118+
}

‎packages/next/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@
252252
"string-hash": "1.1.3",
253253
"string_decoder": "1.3.0",
254254
"strip-ansi": "6.0.0",
255+
"tar": "6.1.11",
255256
"taskr": "1.1.0",
256257
"terser": "5.10.0",
257258
"text-table": "0.2.0",

‎packages/next/taskfile.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1410,6 +1410,16 @@ export async function ncc_nft(task, opts) {
14101410
.ncc({ packageName: '@vercel/nft', externals })
14111411
.target('compiled/@vercel/nft')
14121412
}
1413+
1414+
// eslint-disable-next-line camelcase
1415+
externals['tar'] = 'next/dist/compiled/tar'
1416+
export async function ncc_tar(task, opts) {
1417+
await task
1418+
.source(opts.src || relative(__dirname, require.resolve('tar')))
1419+
.ncc({ packageName: 'tar', externals })
1420+
.target('compiled/tar')
1421+
}
1422+
14131423
// eslint-disable-next-line camelcase
14141424
externals['terser'] = 'next/dist/compiled/terser'
14151425
export async function ncc_terser(task, opts) {
@@ -1729,6 +1739,7 @@ export async function ncc(task, opts) {
17291739
'ncc_string_hash',
17301740
'ncc_strip_ansi',
17311741
'ncc_nft',
1742+
'ncc_tar',
17321743
'ncc_terser',
17331744
'ncc_text_table',
17341745
'ncc_unistore',

0 commit comments

Comments
 (0)