Repeat method to run a test multiple time

packages/vitest/src/node/reporters/json.ts

@@ -10,7 +10,7 @@ import { parseStacktrace } from '../../utils/source-map'
 // the following types are extracted from the Jest repository (and simplified)
 // the commented-out fields are the missing ones
 
-type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled'
+type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' | 'repeated'
 type Milliseconds = number
 interface Callsite { line: number; column: number }
 const StatusMap: Record<TaskState, Status> = {
@@ -20,6 +20,7 @@ const StatusMap: Record<TaskState, Status> = {
   run: 'pending',
   skip: 'skipped',
   todo: 'todo',
+  repeats: 'repeated',
 }
 
 interface FormattedAssertionResult {

packages/vitest/src/node/reporters/renderers/utils.ts

@@ -103,11 +103,13 @@ export function getStateString(tasks: Task[], name = 'tests', showTotal = true)
   const failed = tasks.filter(i => i.result?.state === 'fail')
   const skipped = tasks.filter(i => i.mode === 'skip')
   const todo = tasks.filter(i => i.mode === 'todo')
+  const repeated = tasks.filter(i => i.mode === 'repeats')
 
   return [
     failed.length ? c.bold(c.red(`${failed.length} failed`)) : null,
     passed.length ? c.bold(c.green(`${passed.length} passed`)) : null,
     skipped.length ? c.yellow(`${skipped.length} skipped`) : null,
+    repeated.length ? c.yellow(`${repeated.length} repeated`) : null,
     todo.length ? c.gray(`${todo.length} todo`) : null,
   ].filter(Boolean).join(c.dim(' | ')) + (showTotal ? c.gray(` (${tasks.length})`) : '')
 }

packages/vitest/src/runtime/run.ts

@@ -113,7 +113,7 @@ const callCleanupHooks = async (cleanups: HookCleanupCallback[]) => {
 }
 
 export async function runTest(test: Test) {
-  if (test.mode !== 'run') {
+  if (test.mode !== 'run' && test.mode !== 'repeats') {
     const { getSnapshotClient } = await import('../integrations/snapshot/chai')
     getSnapshotClient().skipTestSnapshots(test)
     return
@@ -145,8 +145,8 @@ export async function runTest(test: Test) {
 
   workerState.current = test
 
-  const retry = test.retry || 1
-  for (let retryCount = 0; retryCount < retry; retryCount++) {
+  const retry = test.mode === 'repeats' ? test.repeats! : test.retry || 1
+  for (let retryCount = test.mode === 'repeats' ? 1 : 0; test.mode === 'repeats' ? retryCount <= retry : retryCount < retry; retryCount++) {
     let beforeEachCleanups: HookCleanupCallback[] = []
     try {
       setState<MatcherState>({
@@ -180,7 +180,10 @@ export async function runTest(test: Test) {
       if (isExpectingAssertions === true && assertionCalls === 0)
         throw isExpectingAssertionsError
 
-      test.result.state = 'pass'
+      if (test.mode === 'run')
+        test.result.state = 'pass'
+      else if (test.mode === 'repeats' && retry === retryCount)
+        test.result.state = 'pass'
     }
     catch (e) {
       const error = processError(e)
@@ -203,6 +206,9 @@ export async function runTest(test: Test) {
     if (test.result.state === 'pass')
       break
 
+    if (test.mode === 'repeats' && test.result.state === 'fail')
+      break
+
     // update retry info
     updateTask(test)
   }

packages/vitest/src/runtime/suite.ts

@@ -61,7 +61,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
     if (!isRunningInTest())
       throw new Error('`test()` and `it()` is only available in test mode.')
 
-    const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run'
+    const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run'
 
     if (typeof options === 'number')
       options = { timeout: options }
@@ -74,6 +74,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       suite: undefined!,
       fails: this.fails,
       retry: options?.retry,
+      // 5 repetitions by default
+      repeats: mode === 'repeats' && !options?.repeats ? 5 : options?.repeats,
     } as Omit<Test, 'context'> as Test
 
     if (this.concurrent || concurrent)
@@ -177,7 +179,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
 
 function createSuite() {
   function suiteFn(this: Record<string, boolean | undefined>, name: string, factory?: SuiteFactory, options?: number | TestOptions) {
-    const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run'
+    const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run'
     return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle, options)
   }
 
@@ -202,14 +204,14 @@ function createSuite() {
   suiteFn.runIf = (condition: any) => (condition ? suite : suite.skip) as SuiteAPI
 
   return createChainable(
-    ['concurrent', 'shuffle', 'skip', 'only', 'todo'],
+    ['concurrent', 'shuffle', 'skip', 'only', 'todo', 'repeats'],
     suiteFn,
   ) as unknown as SuiteAPI
 }
 
 function createTest(fn: (
   (
-    this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>,
+    this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails' | 'repeats', boolean | undefined>,
     title: string,
     fn?: TestFunction,
     options?: number | TestOptions
@@ -239,7 +241,7 @@ function createTest(fn: (
   testFn.runIf = (condition: any) => (condition ? test : test.skip) as TestAPI
 
   return createChainable(
-    ['concurrent', 'skip', 'only', 'todo', 'fails'],
+    ['concurrent', 'skip', 'only', 'todo', 'fails', 'repeats'],
     testFn,
   ) as TestAPI
 }

packages/vitest/src/typecheck/collect.ts

@@ -26,7 +26,7 @@ interface LocalCallDefinition {
   end: number
   name: string
   type: 'suite' | 'test'
-  mode: 'run' | 'skip' | 'only' | 'todo'
+  mode: 'run' | 'skip' | 'only' | 'todo' | 'repeats'
   task: ParsedSuite | ParsedFile | ParsedTest
 }
 

packages/vitest/src/types/tasks.ts

@@ -2,7 +2,7 @@ import type { ChainableFunction } from '../runtime/chain'
 import type { BenchFactory, Benchmark, BenchmarkAPI, BenchmarkResult } from './benchmark'
 import type { Awaitable, ErrorWithDiff, UserConsoleLog } from './general'
 
-export type RunMode = 'run' | 'skip' | 'only' | 'todo'
+export type RunMode = 'run' | 'skip' | 'only' | 'todo' | 'repeats'
 export type TaskState = RunMode | 'pass' | 'fail'
 
 export interface TaskBase {
@@ -17,6 +17,7 @@ export interface TaskBase {
   retry?: number
   logs?: UserConsoleLog[]
   meta?: any
+  repeats?: number
 }
 
 export interface TaskResult {
@@ -33,6 +34,7 @@ export interface TaskResult {
   hooks?: Partial<Record<keyof SuiteHooks, TaskState>>
   benchmark?: BenchmarkResult
   retryCount?: number
+  repeatCount?: number
 }
 
 export type TaskResultPack = [id: string, result: TaskResult | undefined]
@@ -139,7 +141,7 @@ interface TestEachFunction {
 }
 
 type ChainableTestAPI<ExtraContext = {}> = ChainableFunction<
-  'concurrent' | 'only' | 'skip' | 'todo' | 'fails',
+  'concurrent' | 'only' | 'skip' | 'todo' | 'fails' | 'repeats',
   [name: string, fn?: TestFunction<ExtraContext>, options?: number | TestOptions],
   void,
   {
@@ -160,6 +162,12 @@ export interface TestOptions {
    * @default 1
    */
   retry?: number
+  /**
+   * How many times the test will repeat.
+   *
+   * @default 5
+   */
+  repeats?: number
 }
 
 export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {
@@ -169,7 +177,7 @@ export type TestAPI<ExtraContext = {}> = ChainableTestAPI<ExtraContext> & {
 }
 
 type ChainableSuiteAPI<ExtraContext = {}> = ChainableFunction<
-  'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle',
+  'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle' | 'repeats',
   [name: string, factory?: SuiteFactory<ExtraContext>, options?: number | TestOptions],
   SuiteCollector<ExtraContext>,
   {