Add falling back to wasm swc build on native load failure

packages/next/build/swc/index.js

@@ -1,46 +1,81 @@
+import path from 'path'
+import { pathToFileURL } from 'url'
 import { platform, arch } from 'os'
 import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples'
-import { version as nextVersion, optionalDependencies } from 'next/package.json'
 import * as Log from '../output/log'
 import { getParserOptions } from './options'
 import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
 import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
+import { downloadWasmSwc } from '../../lib/download-wasm-swc'
+import { version as nextVersion } from 'next/package.json'
 
 const ArchName = arch()
 const PlatformName = platform()
 const triples = platformArchTriples[PlatformName][ArchName] || []
 
 let nativeBindings
 let wasmBindings
+let downloadWasmPromise
+let pendingBindings
 export const lockfilePatchPromise = {}
 
 async function loadBindings() {
-  if (!lockfilePatchPromise.cur) {
-    // always run lockfile check once so that it gets patched
-    // even if it doesn't fail to load locally
-    lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
-      console.error
-    )
+  if (pendingBindings) {
+    return pendingBindings
   }
+  pendingBindings = new Promise(async (resolve, reject) => {
+    if (!lockfilePatchPromise.cur) {
+      // always run lockfile check once so that it gets patched
+      // even if it doesn't fail to load locally
+      lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
+        console.error
+      )
+    }
 
-  let attempts = []
-  try {
-    return loadNative()
-  } catch (a) {
-    attempts = attempts.concat(a)
-  }
+    let attempts = []
+    try {
+      return resolve(loadNative())
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
 
-  // TODO: fetch wasm and fallback when loading native fails
-  // so that users aren't blocked on this, we still want to
-  // report the native load failure so we can patch though
-  try {
-    let bindings = await loadWasm()
-    return bindings
-  } catch (a) {
-    attempts = attempts.concat(a)
-  }
+    try {
+      let bindings = await loadWasm()
+      eventSwcLoadFailure({ wasm: 'enabled' })
+      return resolve(bindings)
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
 
-  logLoadFailure(attempts)
+    try {
+      // if not installed already download wasm package on-demand
+      // we download to a custom directory instead of to node_modules
+      // as node_module import attempts are cached and can't be re-attempted
+      // x-ref: https://github.com/nodejs/modules/issues/307
+      const wasmDirectory = path.join(
+        path.dirname(require.resolve('next/package.json')),
+        'wasm'
+      )
+      if (!downloadWasmPromise) {
+        downloadWasmPromise = downloadWasmSwc(nextVersion, wasmDirectory)
+      }
+      await downloadWasmPromise
+      let bindings = await loadWasm(pathToFileURL(wasmDirectory).href)
+      eventSwcLoadFailure({ wasm: 'fallback' })
+
+      // still log native load attempts so user is
+      // aware it failed and should be fixed
+      for (const attempt of attempts) {
+        Log.warn(attempt)
+      }
+      return resolve(bindings)
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
+
+    logLoadFailure(attempts, true)
+  })
+  return pendingBindings
 }
 
 function loadBindingsSync() {
@@ -56,47 +91,16 @@ function loadBindingsSync() {
 
 let loggingLoadFailure = false
 
-function logLoadFailure(attempts) {
+function logLoadFailure(attempts, triedWasm = false) {
   // make sure we only emit the event and log the failure once
   if (loggingLoadFailure) return
   loggingLoadFailure = true
 
   for (let attempt of attempts) {
     Log.warn(attempt)
   }
-  let glibcVersion
-  let installedSwcPackages
 
-  try {
-    glibcVersion = process.report?.getReport().header.glibcVersionRuntime
-  } catch (_) {}
-
-  try {
-    const pkgNames = Object.keys(optionalDependencies || {}).filter((pkg) =>
-      pkg.startsWith('@next/swc')
-    )
-    const installedPkgs = []
-
-    for (const pkg of pkgNames) {
-      try {
-        const { version } = require(`${pkg}/package.json`)
-        installedPkgs.push(`${pkg}@${version}`)
-      } catch (_) {}
-    }
-
-    if (installedPkgs.length > 0) {
-      installedSwcPackages = installedPkgs.sort().join(',')
-    }
-  } catch (_) {}
-
-  eventSwcLoadFailure({
-    nextVersion,
-    glibcVersion,
-    installedSwcPackages,
-    arch: process.arch,
-    platform: process.platform,
-    nodeVersion: process.versions.node,
-  })
+  eventSwcLoadFailure({ wasm: triedWasm ? 'failed' : undefined })
     .then(() => lockfilePatchPromise.cur || Promise.resolve())
     .finally(() => {
       Log.error(
@@ -106,15 +110,21 @@ function logLoadFailure(attempts) {
     })
 }
 
-async function loadWasm() {
+async function loadWasm(importPath = '') {
   if (wasmBindings) {
     return wasmBindings
   }
 
   let attempts = []
   for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
     try {
-      let bindings = await import(pkg)
+      let pkgPath = pkg
+
+      if (importPath) {
+        // the import path must be exact when not in node_modules
+        pkgPath = path.join(importPath, pkg, 'wasm.js')
+      }
+      let bindings = await import(pkgPath)
       if (pkg === '@next/swc-wasm-web') {
         bindings = await bindings.default()
       }
@@ -139,14 +149,16 @@ async function loadWasm() {
       }
       return wasmBindings
     } catch (e) {
-      // Do not report attempts to load wasm when it is still experimental
-      // if (e?.code === 'ERR_MODULE_NOT_FOUND') {
-      //   attempts.push(`Attempted to load ${pkg}, but it was not installed`)
-      // } else {
-      //   attempts.push(
-      //     `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
-      //   )
-      // }
+      // Only log attempts for loading wasm when loading as fallback
+      if (importPath) {
+        if (e?.code === 'ERR_MODULE_NOT_FOUND') {
+          attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+        } else {
+          attempts.push(
+            `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
+          )
+        }
+      }
     }
   }
 

packages/next/compiled/tar/LICENSE

@@ -0,0 +1,15 @@
+The ISC License
+
+Copyright (c) Isaac Z. Schlueter and Contributors
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

packages/next/compiled/tar/package.json

@@ -0,0 +1 @@
+{"name":"tar","main":"index.js","author":"Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)","license":"ISC"}

packages/next/lib/download-wasm-swc.ts

@@ -0,0 +1,118 @@
+import os from 'os'
+import fs from 'fs'
+import path from 'path'
+import * as Log from '../build/output/log'
+import { execSync } from 'child_process'
+import tar from 'next/dist/compiled/tar'
+import fetch from 'next/dist/compiled/node-fetch'
+import { fileExists } from './file-exists'
+
+const MAX_VERSIONS_TO_CACHE = 5
+
+export async function downloadWasmSwc(
+  version: string,
+  wasmDirectory: string,
+  variant: 'nodejs' | 'web' = 'nodejs'
+) {
+  const pkgName = `@next/swc-wasm-${variant}`
+  const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
+  const outputDirectory = path.join(wasmDirectory, pkgName)
+
+  if (await fileExists(outputDirectory)) {
+    // if the package is already downloaded a different
+    // failure occurred than not being present
+    return
+  }
+
+  // get platform specific cache directory adapted from playwright's handling
+  // https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
+  const cacheDirectory = (() => {
+    let result
+    const envDefined = process.env['NEXT_SWC_PATH']
+
+    if (envDefined) {
+      result = envDefined
+    } else {
+      let systemCacheDirectory
+      if (process.platform === 'linux') {
+        systemCacheDirectory =
+          process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
+      } else if (process.platform === 'darwin') {
+        systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
+      } else if (process.platform === 'win32') {
+        systemCacheDirectory =
+          process.env.LOCALAPPDATA ||
+          path.join(os.homedir(), 'AppData', 'Local')
+      } else {
+        console.error(new Error('Unsupported platform: ' + process.platform))
+        process.exit(0)
+      }
+      result = path.join(systemCacheDirectory, 'next-swc')
+    }
+
+    if (!path.isAbsolute(result)) {
+      // It is important to resolve to the absolute path:
+      //   - for unzipping to work correctly;
+      //   - so that registry directory matches between installation and execution.
+      // INIT_CWD points to the root of `npm/yarn install` and is probably what
+      // the user meant when typing the relative path.
+      result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
+    }
+    return result
+  })()
+
+  await fs.promises.mkdir(outputDirectory, { recursive: true })
+
+  const extractFromTar = async () => {
+    await tar.x({
+      file: path.join(cacheDirectory, tarFileName),
+      cwd: outputDirectory,
+      strip: 1,
+    })
+  }
+
+  if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
+    Log.info('Downloading WASM swc package...')
+    await fs.promises.mkdir(cacheDirectory, { recursive: true })
+    const tempFile = path.join(
+      cacheDirectory,
+      `${tarFileName}.temp-${Date.now()}`
+    )
+    let registry = `https://registry.npmjs.org/`
+
+    try {
+      const output = execSync('npm config get registry').toString().trim()
+      if (output.startsWith('http')) {
+        registry = output
+      }
+    } catch (_) {}
+
+    await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
+      if (!res.ok) {
+        throw new Error(`request failed with status ${res.status}`)
+      }
+      const cacheWriteStream = fs.createWriteStream(tempFile)
+
+      return new Promise<void>((resolve, reject) => {
+        res.body
+          .pipe(cacheWriteStream)
+          .on('error', (err) => reject(err))
+          .on('finish', () => resolve())
+      }).finally(() => cacheWriteStream.close())
+    })
+    await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
+  }
+  await extractFromTar()
+
+  const cacheFiles = await fs.promises.readdir(cacheDirectory)
+
+  if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
+    cacheFiles.sort()
+
+    for (let i = MAX_VERSIONS_TO_CACHE - 1; i++; i < cacheFiles.length) {
+      await fs.promises
+        .unlink(path.join(cacheDirectory, cacheFiles[i]))
+        .catch(() => {})
+    }
+  }
+}

packages/next/package.json

@@ -252,6 +252,7 @@
     "string-hash": "1.1.3",
     "string_decoder": "1.3.0",
     "strip-ansi": "6.0.0",
+    "tar": "6.1.11",
     "taskr": "1.1.0",
     "terser": "5.10.0",
     "text-table": "0.2.0",

packages/next/taskfile.js

@@ -1410,6 +1410,16 @@ export async function ncc_nft(task, opts) {
     .ncc({ packageName: '@vercel/nft', externals })
     .target('compiled/@vercel/nft')
 }
+
+// eslint-disable-next-line camelcase
+externals['tar'] = 'next/dist/compiled/tar'
+export async function ncc_tar(task, opts) {
+  await task
+    .source(opts.src || relative(__dirname, require.resolve('tar')))
+    .ncc({ packageName: 'tar', externals })
+    .target('compiled/tar')
+}
+
 // eslint-disable-next-line camelcase
 externals['terser'] = 'next/dist/compiled/terser'
 export async function ncc_terser(task, opts) {
@@ -1729,6 +1739,7 @@ export async function ncc(task, opts) {
         'ncc_string_hash',
         'ncc_strip_ansi',
         'ncc_nft',
+        'ncc_tar',
         'ncc_terser',
         'ncc_text_table',
         'ncc_unistore',