From dbac2c812026f8178b1ea2c16e68595cdc39e947 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 12 Aug 2024 22:49:40 +0200 Subject: [PATCH] chore: cache specs --- packages/browser/src/node/plugin.ts | 2 +- packages/browser/src/node/pool.ts | 4 +- packages/browser/src/node/serverTester.ts | 2 +- packages/vitest/src/api/setup.ts | 3 +- packages/vitest/src/node/core.ts | 87 ++++++++++++++----- packages/vitest/src/node/pool.ts | 35 ++------ .../src/node/reporters/renderers/utils.ts | 4 +- packages/vitest/src/node/workspace.ts | 26 ++++-- packages/vitest/src/runtime/types/utils.ts | 1 + packages/vitest/src/utils/test-helpers.ts | 4 +- test/core/test/sequencers.test.ts | 2 +- 11 files changed, 106 insertions(+), 64 deletions(-) diff --git a/packages/browser/src/node/plugin.ts b/packages/browser/src/node/plugin.ts index 749442364baa..dc0b8f096403 100644 --- a/packages/browser/src/node/plugin.ts +++ b/packages/browser/src/node/plugin.ts @@ -157,7 +157,7 @@ export default (browserServer: BrowserServer, base = '/'): Plugin[] => { name: 'vitest:browser:tests', enforce: 'pre', async config() { - const allTestFiles = await project.globTestFiles() + const { testFiles: allTestFiles } = await project.globTestFiles() const browserTestFiles = allTestFiles.filter( file => getFilePoolName(project, file) === 'browser', ) diff --git a/packages/browser/src/node/pool.ts b/packages/browser/src/node/pool.ts index e1a0b5ea61ae..dec4d1310405 100644 --- a/packages/browser/src/node/pool.ts +++ b/packages/browser/src/node/pool.ts @@ -1,7 +1,7 @@ import * as nodeos from 'node:os' import crypto from 'node:crypto' import { relative } from 'pathe' -import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject } from 'vitest/node' +import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject, WorkspaceSpec } from 'vitest/node' import { createDebugger } from 'vitest/node' const debug = createDebugger('vitest:browser:pool') @@ -92,7 +92,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool { await Promise.all(promises) } - const runWorkspaceTests = async (method: 'run' | 'collect', specs: [WorkspaceProject, string][]) => { + const runWorkspaceTests = async (method: 'run' | 'collect', specs: WorkspaceSpec[]) => { const groupedFiles = new Map() for (const [project, file] of specs) { const files = groupedFiles.get(project) || [] diff --git a/packages/browser/src/node/serverTester.ts b/packages/browser/src/node/serverTester.ts index 6dccc013d20f..4e658da8e90c 100644 --- a/packages/browser/src/node/serverTester.ts +++ b/packages/browser/src/node/serverTester.ts @@ -22,7 +22,7 @@ export async function resolveTester( const { contextId, testFile } = server.resolveTesterUrl(url.pathname) const project = server.project const state = server.state - const testFiles = await project.globTestFiles() + const { testFiles } = await project.globTestFiles() // if decoded test file is "__vitest_all__" or not in the list of known files, run all tests const tests = testFile === '__vitest_all__' diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index 2f380a56d00a..b283c0b3a00c 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -103,12 +103,13 @@ export function setup(ctx: Vitest, _server?: ViteDevServer) { }, async getTestFiles() { const spec = await ctx.globTestFiles() - return spec.map(([project, file]) => [ + return spec.map(([project, file, options]) => [ { name: project.config.name, root: project.config.root, }, file, + options, ]) }, }, diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index c940e63a73c7..5987c063360b 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -17,7 +17,7 @@ import { WebSocketReporter } from '../api/setup' import type { SerializedCoverageConfig } from '../runtime/config' import type { SerializedSpec } from '../runtime/types/utils' import type { ArgumentsType, OnServerRestartHandler, ProvidedContext, UserConsoleLog } from '../types/general' -import { createPool } from './pool' +import { createPool, getFilePoolName } from './pool' import type { ProcessPool, WorkspaceSpec } from './pool' import { createBenchmarkReporters, createReporters } from './reporters/utils' import { StateManager } from './state' @@ -77,10 +77,14 @@ export class Vitest { private resolvedProjects: WorkspaceProject[] = [] public projects: WorkspaceProject[] = [] - private projectsTestFiles = new Map>() public distPath!: string + private _cachedSpecs = new Map() + + /** @deprecated use `_cachedSpecs` */ + projectTestFiles = this._cachedSpecs + constructor( public readonly mode: VitestRunMode, options: VitestOptions = {}, @@ -103,7 +107,7 @@ export class Vitest { this.coverageProvider = undefined this.runningPromise = undefined this.distPath = undefined! - this.projectsTestFiles.clear() + this._cachedSpecs.clear() const resolved = resolveConfig(this.mode, options, server.config, this.logger) @@ -202,6 +206,9 @@ export class Vitest { return this.coreWorkspaceProject } + /** + * @deprecated use Reported Task API instead + */ public getProjectByTaskId(taskId: string): WorkspaceProject { const task = this.state.idMap.get(taskId) const projectName = (task as File).projectName || task?.file?.projectName || '' @@ -216,7 +223,7 @@ export class Vitest { || this.projects[0] } - private async getWorkspaceConfigPath() { + private async getWorkspaceConfigPath(): Promise { if (this.config.workspace) { return this.config.workspace } @@ -423,8 +430,8 @@ export class Vitest { } } - private async getTestDependencies(filepath: WorkspaceSpec, deps = new Set()) { - const addImports = async ([project, filepath]: WorkspaceSpec) => { + private async getTestDependencies([project, filepath]: WorkspaceSpec, deps = new Set()) { + const addImports = async (project: WorkspaceProject, filepath: string) => { if (deps.has(filepath)) { return } @@ -440,13 +447,13 @@ export class Vitest { const path = await project.server.pluginContainer.resolveId(dep, filepath, { ssr: true }) const fsPath = path && !path.external && path.id.split('?')[0] if (fsPath && !fsPath.includes('node_modules') && !deps.has(fsPath) && existsSync(fsPath)) { - await addImports([project, fsPath]) + await addImports(project, fsPath) } })) } - await addImports(filepath) - deps.delete(filepath[1]) + await addImports(project, filepath) + deps.delete(filepath) return deps } @@ -500,12 +507,31 @@ export class Vitest { return runningTests } + /** + * @deprecated remove when vscode extension supports "getFileWorkspaceSpecs" + */ getProjectsByTestFile(file: string) { - const projects = this.projectsTestFiles.get(file) - if (!projects) { - return [] + return this.getFileWorkspaceSpecs(file) + } + + getFileWorkspaceSpecs(file: string) { + const _cached = this._cachedSpecs.get(file) + if (_cached) { + return _cached } - return Array.from(projects).map(project => [project, file] as WorkspaceSpec) + + const specs: WorkspaceSpec[] = [] + for (const project of this.projects) { + if (project.isTestFile(file)) { + const pool = getFilePoolName(project, file) + specs.push([project, file, { pool }]) + } + if (project.isTypecheckFile(file)) { + specs.push([project, file, { pool: 'typescript' }]) + } + } + specs.forEach(spec => this.ensureSpecCached(spec)) + return specs } async initializeGlobalSetup(paths: WorkspaceSpec[]) { @@ -538,8 +564,11 @@ export class Vitest { await this.report('onPathsCollected', filepaths) await this.report('onSpecsCollected', specs.map( - ([project, file]) => - [{ name: project.config.name, root: project.config.root }, file] as SerializedSpec, + ([project, file, options]) => + [{ + name: project.config.name, + root: project.config.root, + }, file, options] satisfies SerializedSpec, )) // previous run @@ -856,7 +885,6 @@ export class Vitest { })) if (matchingProjects.length > 0) { - this.projectsTestFiles.set(id, new Set(matchingProjects)) this.changedTests.add(id) this.scheduleRerun([id]) } @@ -1054,17 +1082,32 @@ export class Vitest { public async globTestFiles(filters: string[] = []) { const files: WorkspaceSpec[] = [] await Promise.all(this.projects.map(async (project) => { - const specs = await project.globTestFiles(filters) - specs.forEach((file) => { - files.push([project, file]) - const projects = this.projectsTestFiles.get(file) || new Set() - projects.add(project) - this.projectsTestFiles.set(file, projects) + const { testFiles, typecheckTestFiles } = await project.globTestFiles(filters) + testFiles.forEach((file) => { + const pool = getFilePoolName(project, file) + const spec: WorkspaceSpec = [project, file, { pool }] + this.ensureSpecCached(spec) + files.push(spec) + }) + typecheckTestFiles.forEach((file) => { + const spec: WorkspaceSpec = [project, file, { pool: 'typecheck' }] + this.ensureSpecCached(spec) + files.push(spec) }) })) return files } + private ensureSpecCached(spec: WorkspaceSpec) { + const file = spec[1] + const specs = this._cachedSpecs.get(file) || [] + const included = specs.some(_s => _s[0] === spec[0] && _s[2].pool === spec[2].pool) + if (!included) { + specs.push(spec) + this._cachedSpecs.set(file, specs) + } + } + // The server needs to be running for communication shouldKeepServer() { return !!this.config?.watch diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index da26d40cafa5..2e225e0e5053 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -10,7 +10,7 @@ import type { WorkspaceProject } from './workspace' import { createTypecheckPool } from './pools/typecheck' import { createVmForksPool } from './pools/vmForks' -export type WorkspaceSpec = [project: WorkspaceProject, testFile: string] +export type WorkspaceSpec = [project: WorkspaceProject, testFile: string, options: { pool: Pool }] export type RunWithFiles = ( files: WorkspaceSpec[], invalidates?: string[] @@ -39,16 +39,11 @@ export const builtinPools: BuiltinPool[] = [ 'typescript', ] -function getDefaultPoolName(project: WorkspaceProject, file: string): Pool | null { - for (const glob of project.config.include) { - if (mm.isMatch(file, glob, { cwd: project.config.root, ignore: project.config.exclude })) { - if (project.config.browser.enabled) { - return 'browser' - } - return project.config.pool - } +function getDefaultPoolName(project: WorkspaceProject): Pool { + if (project.config.browser.enabled) { + return 'browser' } - return null + return project.config.pool } export function getFilePoolName(project: WorkspaceProject, file: string) { @@ -62,7 +57,7 @@ export function getFilePoolName(project: WorkspaceProject, file: string) { return pool as Pool } } - return getDefaultPoolName(project, file) + return getDefaultPoolName(project) } export function createPool(ctx: Vitest): ProcessPool { @@ -170,21 +165,9 @@ export function createPool(ctx: Vitest): ProcessPool { } for (const spec of files) { - const [project, file] = spec - const pool = getFilePoolName(project, file) - if (pool != null) { - filesByPool[pool] ??= [] - filesByPool[pool].push(spec) - } - - if (project.config.typecheck.enabled) { - for (const glob of project.config.typecheck.include) { - if (mm.isMatch(file, glob, { cwd: project.config.root, ignore: project.config.typecheck.exclude })) { - filesByPool.typescript ??= [] - filesByPool.typescript.push(spec) - } - } - } + const { pool } = spec[2] + filesByPool[pool] ??= [] + filesByPool[pool].push(spec) } const Sequencer = ctx.config.sequence.sequencer diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 9c8212944a70..4422ec0dd9d7 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -254,6 +254,6 @@ export function formatProjectName(name: string | undefined, suffix = ' ') { const index = name .split('') .reduce((acc, v, idx) => acc + v.charCodeAt(0) + idx, 0) - const colors = [c.blue, c.yellow, c.cyan, c.green, c.magenta] - return colors[index % colors.length](`|${name}|`) + suffix + const colors = [c.bgBlue, c.bgYellow, c.bgCyan, c.bgGreen, c.bgMagenta] + return colors[index % colors.length](` ${c.white(name)} `) + suffix } diff --git a/packages/vitest/src/node/workspace.ts b/packages/vitest/src/node/workspace.ts index 408f1da3f870..494c3147e4cf 100644 --- a/packages/vitest/src/node/workspace.ts +++ b/packages/vitest/src/node/workspace.ts @@ -97,6 +97,7 @@ export class WorkspaceProject { closingPromise: Promise | undefined testFilesList: string[] | null = null + typecheckFilesList: string[] | null = null public testProject!: TestProject @@ -225,15 +226,24 @@ export class WorkspaceProject { ? [] : this.globAllTestFiles(include, exclude, includeSource, dir), typecheck.enabled - ? this.globFiles(typecheck.include, typecheck.exclude, dir) + ? (this.typecheckFilesList || this.globFiles(typecheck.include, typecheck.exclude, dir)) : [], ]) - return this.filterFiles( - [...testFiles, ...typecheckTestFiles], - filters, - dir, - ) + this.typecheckFilesList = typecheckTestFiles + + return { + testFiles: this.filterFiles( + testFiles, + filters, + dir, + ), + typecheckTestFiles: this.filterFiles( + typecheckTestFiles, + filters, + dir, + ), + } } async globAllTestFiles( @@ -275,6 +285,10 @@ export class WorkspaceProject { return this.testFilesList && this.testFilesList.includes(id) } + isTypecheckFile(id: string) { + return this.typecheckFilesList && this.typecheckFilesList.includes(id) + } + async globFiles(include: string[], exclude: string[], cwd: string) { const globOptions: fg.Options = { dot: true, diff --git a/packages/vitest/src/runtime/types/utils.ts b/packages/vitest/src/runtime/types/utils.ts index e65c04ce3382..7e76bcb08fc6 100644 --- a/packages/vitest/src/runtime/types/utils.ts +++ b/packages/vitest/src/runtime/types/utils.ts @@ -1,4 +1,5 @@ export type SerializedSpec = [ project: { name: string | undefined; root: string }, file: string, + options: { pool: string }, ] diff --git a/packages/vitest/src/utils/test-helpers.ts b/packages/vitest/src/utils/test-helpers.ts index dcfaef619e93..fd24f3135911 100644 --- a/packages/vitest/src/utils/test-helpers.ts +++ b/packages/vitest/src/utils/test-helpers.ts @@ -1,8 +1,8 @@ import { promises as fs } from 'node:fs' import mm from 'micromatch' -import type { WorkspaceProject } from '../node/workspace' import type { EnvironmentOptions, TransformModePatterns, VitestEnvironment } from '../node/types/config' import type { ContextTestEnvironment } from '../types/worker' +import type { WorkspaceSpec } from '../node/pool' import { groupBy } from './base' export const envsOrder = ['node', 'jsdom', 'happy-dom', 'edge-runtime'] @@ -27,7 +27,7 @@ function getTransformMode( } export async function groupFilesByEnv( - files: (readonly [WorkspaceProject, string])[], + files: Array, ) { const filesWithEnv = await Promise.all( files.map(async ([project, file]) => { diff --git a/test/core/test/sequencers.test.ts b/test/core/test/sequencers.test.ts index 448b1515b72b..74f13181551a 100644 --- a/test/core/test/sequencers.test.ts +++ b/test/core/test/sequencers.test.ts @@ -26,7 +26,7 @@ function buildWorkspace() { const workspace = buildWorkspace() function workspaced(files: string[]) { - return files.map(file => [workspace, file] as WorkspaceSpec) + return files.map(file => [workspace, file, { pool: 'forks' }] satisfies WorkspaceSpec) } describe('base sequencer', () => {