diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 31aa979f37db..59d832475332 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -10,7 +10,7 @@ import { parseStacktrace } from '../../utils/source-map' // the following types are extracted from the Jest repository (and simplified) // the commented-out fields are the missing ones -type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' +type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' | 'repeated' type Milliseconds = number interface Callsite { line: number; column: number } const StatusMap: Record = { @@ -20,6 +20,7 @@ const StatusMap: Record = { run: 'pending', skip: 'skipped', todo: 'todo', + repeats: 'repeated', } interface FormattedAssertionResult { diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 8d0313677da6..9d5e6671dbcb 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -103,11 +103,13 @@ export function getStateString(tasks: Task[], name = 'tests', showTotal = true) const failed = tasks.filter(i => i.result?.state === 'fail') const skipped = tasks.filter(i => i.mode === 'skip') const todo = tasks.filter(i => i.mode === 'todo') + const repeated = tasks.filter(i => i.mode === 'repeats') return [ failed.length ? c.bold(c.red(`${failed.length} failed`)) : null, passed.length ? c.bold(c.green(`${passed.length} passed`)) : null, skipped.length ? c.yellow(`${skipped.length} skipped`) : null, + repeated.length ? c.yellow(`${repeated.length} repeated`) : null, todo.length ? c.gray(`${todo.length} todo`) : null, ].filter(Boolean).join(c.dim(' | ')) + (showTotal ? c.gray(` (${tasks.length})`) : '') } diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index 45164b75041d..e75d14838118 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -113,7 +113,7 @@ const callCleanupHooks = async (cleanups: HookCleanupCallback[]) => { } export async function runTest(test: Test) { - if (test.mode !== 'run') { + if (test.mode !== 'run' && test.mode !== 'repeats') { const { getSnapshotClient } = await import('../integrations/snapshot/chai') getSnapshotClient().skipTestSnapshots(test) return @@ -145,8 +145,8 @@ export async function runTest(test: Test) { workerState.current = test - const retry = test.retry || 1 - for (let retryCount = 0; retryCount < retry; retryCount++) { + const retry = test.mode === 'repeats' ? test.repeats! : test.retry || 1 + for (let retryCount = test.mode === 'repeats' ? 1 : 0; test.mode === 'repeats' ? retryCount <= retry : retryCount < retry; retryCount++) { let beforeEachCleanups: HookCleanupCallback[] = [] try { setState({ @@ -180,7 +180,10 @@ export async function runTest(test: Test) { if (isExpectingAssertions === true && assertionCalls === 0) throw isExpectingAssertionsError - test.result.state = 'pass' + if (test.mode === 'run') + test.result.state = 'pass' + else if (test.mode === 'repeats' && retry === retryCount) + test.result.state = 'pass' } catch (e) { const error = processError(e) @@ -203,6 +206,9 @@ export async function runTest(test: Test) { if (test.result.state === 'pass') break + if (test.mode === 'repeats' && test.result.state === 'fail') + break + // update retry info updateTask(test) } diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 8a69f6bc05e1..548245bef1ef 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -61,7 +61,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m if (!isRunningInTest()) throw new Error('`test()` and `it()` is only available in test mode.') - const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' + const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' if (typeof options === 'number') options = { timeout: options } @@ -74,6 +74,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m suite: undefined!, fails: this.fails, retry: options?.retry, + // 5 repetitions by default + repeats: mode === 'repeats' && !options?.repeats ? 5 : options?.repeats, } as Omit as Test if (this.concurrent || concurrent) @@ -177,7 +179,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { function suiteFn(this: Record, name: string, factory?: SuiteFactory, options?: number | TestOptions) { - const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' + const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle, options) } @@ -202,14 +204,14 @@ function createSuite() { suiteFn.runIf = (condition: any) => (condition ? suite : suite.skip) as SuiteAPI return createChainable( - ['concurrent', 'shuffle', 'skip', 'only', 'todo'], + ['concurrent', 'shuffle', 'skip', 'only', 'todo', 'repeats'], suiteFn, ) as unknown as SuiteAPI } function createTest(fn: ( ( - this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>, + this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails' | 'repeats', boolean | undefined>, title: string, fn?: TestFunction, options?: number | TestOptions @@ -239,7 +241,7 @@ function createTest(fn: ( testFn.runIf = (condition: any) => (condition ? test : test.skip) as TestAPI return createChainable( - ['concurrent', 'skip', 'only', 'todo', 'fails'], + ['concurrent', 'skip', 'only', 'todo', 'fails', 'repeats'], testFn, ) as TestAPI } diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index 0916b499be13..c09e182ef351 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -26,7 +26,7 @@ interface LocalCallDefinition { end: number name: string type: 'suite' | 'test' - mode: 'run' | 'skip' | 'only' | 'todo' + mode: 'run' | 'skip' | 'only' | 'todo' | 'repeats' task: ParsedSuite | ParsedFile | ParsedTest } diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index da62441943ce..f8e613977786 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -2,7 +2,7 @@ import type { ChainableFunction } from '../runtime/chain' import type { BenchFactory, Benchmark, BenchmarkAPI, BenchmarkResult } from './benchmark' import type { Awaitable, ErrorWithDiff, UserConsoleLog } from './general' -export type RunMode = 'run' | 'skip' | 'only' | 'todo' +export type RunMode = 'run' | 'skip' | 'only' | 'todo' | 'repeats' export type TaskState = RunMode | 'pass' | 'fail' export interface TaskBase { @@ -17,6 +17,7 @@ export interface TaskBase { retry?: number logs?: UserConsoleLog[] meta?: any + repeats?: number } export interface TaskResult { @@ -33,6 +34,7 @@ export interface TaskResult { hooks?: Partial> benchmark?: BenchmarkResult retryCount?: number + repeatCount?: number } export type TaskResultPack = [id: string, result: TaskResult | undefined] @@ -139,7 +141,7 @@ interface TestEachFunction { } type ChainableTestAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'fails', + 'concurrent' | 'only' | 'skip' | 'todo' | 'fails' | 'repeats', [name: string, fn?: TestFunction, options?: number | TestOptions], void, { @@ -160,6 +162,12 @@ export interface TestOptions { * @default 1 */ retry?: number + /** + * How many times the test will repeat. + * + * @default 5 + */ + repeats?: number } export type TestAPI = ChainableTestAPI & { @@ -169,7 +177,7 @@ export type TestAPI = ChainableTestAPI & { } type ChainableSuiteAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle', + 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle' | 'repeats', [name: string, factory?: SuiteFactory, options?: number | TestOptions], SuiteCollector, {