feat: add an option to enable Vite optimizer (#2912)

Author: sheremet-va

docs/config/index.md

@@ -91,6 +91,28 @@ Files to exclude from the test run, using glob pattern.
 
 Handling for dependencies resolution.
 
+#### deps.experimentalOptimizer
+
+- **Type:** `DepOptimizationConfig & { enabled: boolean }`
+- **Version:** Vitets 0.29.0
+- **See also:** [Dep Optimization Options](https://vitejs.dev/config/dep-optimization-options.html)
+
+Enable dependency optimization. If you have a lot of tests, this might improve their performance.
+
+For `jsdom` and `happy-dom` environments, when Vitest will encounter the external library, it will be bundled into a single file using esbuild and imported as a whole module. This is good for several reasons:
+
+- Importing packages with a lot of imports is expensive. By bundling them into one file we can save a lot of time
+- Importing UI libraries is expensive because they are not meant to run inside Node.js
+- Your `alias` configuration is now respected inside bundled packages
+
+You can opt-out of this behavior for certain packages with `exclude` option. You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs.
+
+This options also inherits your `optimizeDeps` configuration. If you redefine `include`/`exclude`/`entries` option in `deps.experimentalOptimizer` it will overwrite your `optimizeDeps` when running tests.
+
+:::note
+You will not be able to edit your `node_modules` code for debugging, since the code is actually located in your `cacheDir` or `test.cache.dir` directory. If you want to debug with `console.log` statements, edit it directly or force rebundling with `deps.experimentalOptimizer.force` option.
+:::
+
 #### deps.external
 
 - **Type:** `(string | RegExp)[]`

packages/vite-node/src/client.ts

@@ -204,18 +204,18 @@ export class ViteNodeRunner {
     return !isInternalRequest(id) && !isNodeBuiltin(id)
   }
 
-  private async _resolveUrl(id: string, importee?: string): Promise<[url: string, fsPath: string]> {
+  private async _resolveUrl(id: string, importer?: string): Promise<[url: string, fsPath: string]> {
     // we don't pass down importee here, because otherwise Vite doesn't resolve it correctly
     // should be checked before normalization, because it removes this prefix
-    if (importee && id.startsWith(VALID_ID_PREFIX))
-      importee = undefined
+    if (importer && id.startsWith(VALID_ID_PREFIX))
+      importer = undefined
     id = normalizeRequestId(id, this.options.base)
     if (!this.shouldResolveId(id))
       return [id, id]
     const { path, exists } = toFilePath(id, this.root)
     if (!this.options.resolveId || exists)
       return [id, path]
-    const resolved = await this.options.resolveId(id, importee)
+    const resolved = await this.options.resolveId(id, importer)
     const resolvedId = resolved
       ? normalizeRequestId(resolved.id, this.options.base)
       : id

packages/vite-node/src/externalize.ts

@@ -105,6 +105,10 @@ async function _shouldExternalize(
 
   id = patchWindowsImportPath(id)
 
+  // always externalize Vite deps, they are too big to inline
+  if (options?.cacheDir && id.includes(options.cacheDir))
+    return id
+
   if (matchExternalizePattern(id, options?.inline))
     return false
   if (matchExternalizePattern(id, options?.external))

packages/vite-node/src/server.ts

@@ -1,5 +1,6 @@
 import { performance } from 'node:perf_hooks'
-import { resolve } from 'pathe'
+import { existsSync } from 'node:fs'
+import { join, relative, resolve } from 'pathe'
 import type { TransformResult, ViteDevServer } from 'vite'
 import createDebug from 'debug'
 import type { EncodedSourceMap } from '@jridgewell/trace-mapping'
@@ -17,6 +18,8 @@ export class ViteNodeServer {
   private fetchPromiseMap = new Map<string, Promise<FetchResult>>()
   private transformPromiseMap = new Map<string, Promise<TransformResult | null | undefined>>()
 
+  private existingOptimizedDeps = new Set<string>()
+
   fetchCache = new Map<string, {
     duration?: number
     timestamp: number
@@ -34,9 +37,12 @@ export class ViteNodeServer {
     // eslint-disable-next-line @typescript-eslint/prefer-ts-expect-error
     // @ts-ignore ssr is not typed in Vite 2, but defined in Vite 3, so we can't use expect-error
     const ssrOptions = server.config.ssr
-    if (ssrOptions) {
-      options.deps ??= {}
 
+    options.deps ??= {}
+
+    options.deps.cacheDir = relative(server.config.root, server.config.cacheDir)
+
+    if (ssrOptions) {
       // we don't externalize ssr, because it has different semantics in Vite
       // if (ssrOptions.external) {
       //   options.deps.external ??= []
@@ -65,10 +71,26 @@ export class ViteNodeServer {
     return shouldExternalize(id, this.options.deps, this.externalizeCache)
   }
 
-  async resolveId(id: string, importer?: string): Promise<ViteNodeResolveId | null> {
+  private async ensureExists(id: string): Promise<boolean> {
+    if (this.existingOptimizedDeps.has(id))
+      return true
+    if (existsSync(id)) {
+      this.existingOptimizedDeps.add(id)
+      return true
+    }
+    return new Promise<boolean>((resolve) => {
+      setTimeout(() => {
+        this.ensureExists(id).then(() => {
+          resolve(true)
+        })
+      })
+    })
+  }
+
+  async resolveId(id: string, importer?: string, transformMode?: 'web' | 'ssr'): Promise<ViteNodeResolveId | null> {
     if (importer && !importer.startsWith(this.server.config.root))
       importer = resolve(this.server.config.root, importer)
-    const mode = (importer && this.getTransformMode(importer)) || 'ssr'
+    const mode = transformMode ?? ((importer && this.getTransformMode(importer)) || 'ssr')
     return this.server.pluginContainer.resolveId(id, importer, { ssr: mode === 'ssr' })
   }
 
@@ -80,12 +102,12 @@ export class ViteNodeServer {
     return (ssrTransformResult?.map || null) as unknown as EncodedSourceMap | null
   }
 
-  async fetchModule(id: string): Promise<FetchResult> {
+  async fetchModule(id: string, transformMode?: 'web' | 'ssr'): Promise<FetchResult> {
     id = normalizeModuleId(id)
     // reuse transform for concurrent requests
     if (!this.fetchPromiseMap.has(id)) {
       this.fetchPromiseMap.set(id,
-        this._fetchModule(id)
+        this._fetchModule(id, transformMode)
           .then((r) => {
             return this.options.sourcemap !== true ? { ...r, map: undefined } : r
           })
@@ -123,9 +145,20 @@ export class ViteNodeServer {
     return 'web'
   }
 
-  private async _fetchModule(id: string): Promise<FetchResult> {
+  private async _fetchModule(id: string, transformMode?: 'web' | 'ssr'): Promise<FetchResult> {
     let result: FetchResult
 
+    const cacheDir = this.options.deps?.cacheDir
+
+    if (cacheDir && id.includes(cacheDir) && !id.includes(this.server.config.root)) {
+      id = join(this.server.config.root, id)
+      const timeout = setTimeout(() => {
+        throw new Error(`ViteNodeServer: ${id} not found. This is a bug, please report it.`)
+      }, 5000) // CI can be quite slow
+      await this.ensureExists(id)
+      clearTimeout(timeout)
+    }
+
     const { path: filePath } = toFilePath(id, this.server.config.root)
 
     const module = this.server.moduleGraph.getModuleById(id)
@@ -143,7 +176,7 @@ export class ViteNodeServer {
     }
     else {
       const start = performance.now()
-      const r = await this._transformRequest(id)
+      const r = await this._transformRequest(id, transformMode)
       duration = performance.now() - start
       result = { code: r?.code, map: r?.map as any }
     }
@@ -157,7 +190,7 @@ export class ViteNodeServer {
     return result
   }
 
-  private async _transformRequest(id: string) {
+  private async _transformRequest(id: string, customTransformMode?: 'web' | 'ssr') {
     debugRequest(id)
 
     let result: TransformResult | null = null
@@ -168,7 +201,9 @@ export class ViteNodeServer {
         return result
     }
 
-    if (this.getTransformMode(id) === 'web') {
+    const transformMode = customTransformMode ?? this.getTransformMode(id)
+
+    if (transformMode === 'web') {
       // for components like Vue, we want to use the client side
       // plugins but then convert the code to be consumed by the server
       result = await this.server.transformRequest(id)

packages/vite-node/src/types.ts

@@ -8,6 +8,7 @@ export type Arrayable<T> = T | Array<T>
 export interface DepsHandlingOptions {
   external?: (string | RegExp)[]
   inline?: (string | RegExp)[] | true
+  cacheDir?: string
   /**
    * Try to guess the CJS version of a package when it's invalid ESM
    * @default false

packages/vitest/src/node/core.ts

@@ -155,8 +155,9 @@ export class Vitest {
   }
 
   async typecheck(filters: string[] = []) {
+    const { dir, root } = this.config
     const { include, exclude } = this.config.typecheck
-    const testsFilesList = await this.globFiles(filters, include, exclude)
+    const testsFilesList = this.filterFiles(await this.globFiles(include, exclude, dir || root), filters)
     const checker = new Typechecker(this, testsFilesList)
     this.typechecker = checker
     checker.onParseEnd(async ({ files, sourceErrors }) => {
@@ -606,32 +607,26 @@ export class Vitest {
     )))
   }
 
-  async globFiles(filters: string[], include: string[], exclude: string[]) {
+  async globFiles(include: string[], exclude: string[], cwd: string) {
     const globOptions: fg.Options = {
       absolute: true,
       dot: true,
-      cwd: this.config.dir || this.config.root,
+      cwd,
       ignore: exclude,
     }
 
-    let testFiles = await fg(include, globOptions)
-
-    if (filters.length && process.platform === 'win32')
-      filters = filters.map(f => toNamespacedPath(f))
-
-    if (filters.length)
-      testFiles = testFiles.filter(i => filters.some(f => i.includes(f)))
-
-    return testFiles
+    return fg(include, globOptions)
   }
 
-  async globTestFiles(filters: string[] = []) {
-    const { include, exclude, includeSource } = this.config
+  private _allTestsCache: string[] | null = null
+
+  async globAllTestFiles(config: ResolvedConfig, cwd: string) {
+    const { include, exclude, includeSource } = config
 
-    const testFiles = await this.globFiles(filters, include, exclude)
+    const testFiles = await this.globFiles(include, exclude, cwd)
 
     if (includeSource) {
-      const files = await this.globFiles(filters, includeSource, exclude)
+      const files = await this.globFiles(includeSource, exclude, cwd)
 
       await Promise.all(files.map(async (file) => {
         try {
@@ -645,9 +640,31 @@ export class Vitest {
       }))
     }
 
+    this._allTestsCache = testFiles
+
+    return testFiles
+  }
+
+  filterFiles(testFiles: string[], filters: string[] = []) {
+    if (filters.length && process.platform === 'win32')
+      filters = filters.map(f => toNamespacedPath(f))
+
+    if (filters.length)
+      return testFiles.filter(i => filters.some(f => i.includes(f)))
+
     return testFiles
   }
 
+  async globTestFiles(filters: string[] = []) {
+    const { dir, root } = this.config
+
+    const testFiles = this._allTestsCache ?? await this.globAllTestFiles(this.config, dir || root)
+
+    this._allTestsCache = null
+
+    return this.filterFiles(testFiles, filters)
+  }
+
   async isTargetFile(id: string, source?: string): Promise<boolean> {
     const relativeId = relative(this.config.dir || this.config.root, id)
     if (mm.isMatch(relativeId, this.config.exclude))

packages/vitest/src/node/create.ts

@@ -27,7 +27,8 @@ export async function createVitest(mode: VitestRunMode, options: UserConfig, vit
 
   const server = await createServer(mergeConfig(config, mergeConfig(viteOverrides, { root: options.root })))
 
-  if (ctx.config.api?.port)
+  // optimizer needs .listen() to be called
+  if (ctx.config.api?.port || ctx.config.deps?.experimentalOptimizer?.enabled)
     await server.listen()
   else
     await server.pluginContainer.buildStart({})

packages/vitest/src/node/plugins/index.ts

@@ -33,7 +33,7 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t
       options() {
         this.meta.watchMode = false
       },
-      config(viteConfig: any) {
+      async config(viteConfig: any) {
         // preliminary merge of options to be able to create server options for vite
         // however to allow vitest plugins to modify vitest config values
         // this is repeated in configResolved where the config is final
@@ -131,15 +131,37 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('t
         }
 
         if (!options.browser) {
-          // disable deps optimization
-          Object.assign(config, {
-            cacheDir: undefined,
-            optimizeDeps: {
+          const optimizeConfig: Partial<ViteConfig> = {}
+          const optimizer = preOptions.deps?.experimentalOptimizer
+          if (!optimizer?.enabled) {
+            optimizeConfig.cacheDir = undefined
+            optimizeConfig.optimizeDeps = {
               // experimental in Vite >2.9.2, entries remains to help with older versions
               disabled: true,
               entries: [],
-            },
-          })
+            }
+          }
+          else {
+            const entries = await ctx.globAllTestFiles(preOptions as ResolvedConfig, preOptions.dir || getRoot())
+            optimizeConfig.cacheDir = preOptions.cache?.dir ?? 'node_modules/.vitest'
+            optimizeConfig.optimizeDeps = {
+              ...viteConfig.optimizeDeps,
+              ...optimizer,
+              disabled: false,
+              entries: [...(optimizer.entries || viteConfig.optimizeDeps?.entries || []), ...entries],
+              exclude: ['vitest', ...(optimizer.exclude || viteConfig.optimizeDeps?.exclude || [])],
+              include: (optimizer.include || viteConfig.optimizeDeps?.include || []).filter((n: string) => n !== 'vitest'),
+            }
+            // Vite throws an error that it cannot rename "deps_temp", but optimization still works
+            // let's not show this error to users
+            const { error: logError } = console
+            console.error = (...args) => {
+              if (typeof args[0] === 'string' && args[0].includes('/deps_temp'))
+                return
+              return logError(...args)
+            }
+          }
+          Object.assign(config, optimizeConfig)
         }
 
         return config

packages/vitest/src/node/pool.ts

@@ -8,7 +8,7 @@ import { createBirpc } from 'birpc'
 import type { RawSourceMap } from 'vite-node'
 import type { ResolvedConfig, WorkerContext, WorkerRPC, WorkerTestEnvironment } from '../types'
 import { distDir, rootDir } from '../constants'
-import { AggregateError, groupBy } from '../utils'
+import { AggregateError, getEnvironmentTransformMode, groupBy } from '../utils'
 import { envsOrder, groupFilesByEnv } from '../utils/test-helpers'
 import type { Vitest } from './core'
 
@@ -195,11 +195,13 @@ function createChannel(ctx: Vitest) {
         const r = await ctx.vitenode.transformRequest(id)
         return r?.map as RawSourceMap | undefined
       },
-      fetch(id) {
-        return ctx.vitenode.fetchModule(id)
+      fetch(id, environment) {
+        const transformMode = getEnvironmentTransformMode(ctx.config, environment)
+        return ctx.vitenode.fetchModule(id, transformMode)
       },
-      resolveId(id, importer) {
-        return ctx.vitenode.resolveId(id, importer)
+      resolveId(id, importer, environment) {
+        const transformMode = getEnvironmentTransformMode(ctx.config, environment)
+        return ctx.vitenode.resolveId(id, importer, transformMode)
       },
       onPathsCollected(paths) {
         ctx.state.collectPaths(paths)

packages/vitest/src/runtime/execute.ts

@@ -43,10 +43,10 @@ export class VitestExecutor extends ViteNodeRunner {
     return environment === 'node' ? !isNodeBuiltin(id) : !id.startsWith('node:')
   }
 
-  async resolveUrl(id: string, importee?: string) {
-    if (importee && importee.startsWith('mock:'))
-      importee = importee.slice(5)
-    return super.resolveUrl(id, importee)
+  async resolveUrl(id: string, importer?: string) {
+    if (importer && importer.startsWith('mock:'))
+      importer = importer.slice(5)
+    return super.resolveUrl(id, importer)
   }
 
   async dependencyRequest(id: string, fsPath: string, callstack: string[]): Promise<any> {