feat(coverage): automatic threshold updating (#2886)

Closes #1241

Author: AriPerkkio

docs/config/index.md

@@ -789,6 +789,16 @@ Do not show files with 100% statement, branch, and function coverage.
 Check thresholds per file.
 See `lines`, `functions`, `branches` and `statements` for the actual thresholds.
 
+#### thresholdAutoUpdate
+
+- **Type:** `boolean`
+- **Default:** `false`
+- **Available for providers:** `'c8' | 'istanbul'`
+- **CLI:** `--coverage.thresholdAutoUpdate=<boolean>`
+
+Update threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds.
+This option helps to maintain thresholds when coverage is improved.
+
 #### lines
 
 - **Type:** `number`

packages/coverage-c8/rollup.config.js

@@ -21,6 +21,7 @@ const external = [
   'vitest',
   'vitest/node',
   'vitest/config',
+  'vitest/coverage',
 ]
 
 const plugins = [

packages/coverage-c8/src/provider.ts

@@ -6,6 +6,7 @@ import c from 'picocolors'
 import { provider } from 'std-env'
 import type { EncodedSourceMap } from 'vite-node'
 import { coverageConfigDefaults } from 'vitest/config'
+import { BaseCoverageProvider } from 'vitest/coverage'
 // eslint-disable-next-line no-restricted-imports
 import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest'
 import type { Vitest } from 'vitest/node'
@@ -17,16 +18,36 @@ import { checkCoverages } from 'c8/lib/commands/check-coverage.js'
 
 type Options = ResolvedCoverageOptions<'c8'>
 
-export class C8CoverageProvider implements CoverageProvider {
+export class C8CoverageProvider extends BaseCoverageProvider implements CoverageProvider {
   name = 'c8'
 
   ctx!: Vitest
   options!: Options
   coverages: Profiler.TakePreciseCoverageReturnType[] = []
 
   initialize(ctx: Vitest) {
+    const config: CoverageC8Options = ctx.config.coverage
+
     this.ctx = ctx
-    this.options = resolveC8Options(ctx.config.coverage, ctx.config.root)
+    this.options = {
+      ...coverageConfigDefaults,
+
+      // Provider specific defaults
+      excludeNodeModules: true,
+      allowExternal: false,
+
+      // User's options
+      ...config,
+
+      // Resolved fields
+      provider: 'c8',
+      reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter),
+      reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory),
+      lines: config['100'] ? 100 : config.lines,
+      functions: config['100'] ? 100 : config.functions,
+      branches: config['100'] ? 100 : config.branches,
+      statements: config['100'] ? 100 : config.statements,
+    }
   }
 
   resolveOptions() {
@@ -156,55 +177,18 @@ export class C8CoverageProvider implements CoverageProvider {
 
     await report.run()
     await checkCoverages(options, report)
-  }
-}
-
-function resolveC8Options(options: CoverageC8Options, root: string): Options {
-  const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory)
-
-  const resolved: Options = {
-    ...coverageConfigDefaults,
-
-    // Provider specific defaults
-    excludeNodeModules: true,
-    allowExternal: false,
-
-    // User's options
-    ...options,
-
-    // Resolved fields
-    provider: 'c8',
-    reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter),
-    reportsDirectory,
-  }
 
-  if (options['100']) {
-    resolved.lines = 100
-    resolved.functions = 100
-    resolved.branches = 100
-    resolved.statements = 100
-  }
-
-  return resolved
-}
-
-function resolveReporters(configReporters: NonNullable<CoverageC8Options['reporter']>): Options['reporter'] {
-  // E.g. { reporter: "html" }
-  if (!Array.isArray(configReporters))
-    return [[configReporters, {}]]
-
-  const resolvedReporters: Options['reporter'] = []
-
-  for (const reporter of configReporters) {
-    if (Array.isArray(reporter)) {
-      // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
-      resolvedReporters.push([reporter[0], reporter[1] || {}])
-    }
-    else {
-      // E.g. { reporter: ["html", "json"]}
-      resolvedReporters.push([reporter, {}])
+    if (this.options.thresholdAutoUpdate && allTestsRun) {
+      this.updateThresholds({
+        coverageMap: await report.getCoverageMapFromAllCoverageFiles(),
+        thresholds: {
+          branches: this.options.branches,
+          functions: this.options.functions,
+          lines: this.options.lines,
+          statements: this.options.statements,
+        },
+        configurationFile: this.ctx.server.config.configFile,
+      })
     }
   }
-
-  return resolvedReporters
 }

packages/coverage-istanbul/rollup.config.js

@@ -19,6 +19,7 @@ const external = [
   'vitest',
   'vitest/node',
   'vitest/config',
+  'vitest/coverage',
 ]
 
 const plugins = [

packages/coverage-istanbul/src/provider.ts

@@ -4,6 +4,7 @@ import { relative, resolve } from 'pathe'
 import type { TransformPluginContext } from 'rollup'
 import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
 import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config'
+import { BaseCoverageProvider } from 'vitest/coverage'
 import libReport from 'istanbul-lib-report'
 import reports from 'istanbul-reports'
 import type { CoverageMap } from 'istanbul-lib-coverage'
@@ -31,7 +32,7 @@ interface TestExclude {
   }
 }
 
-export class IstanbulCoverageProvider implements CoverageProvider {
+export class IstanbulCoverageProvider extends BaseCoverageProvider implements CoverageProvider {
   name = 'istanbul'
 
   ctx!: Vitest
@@ -48,8 +49,20 @@ export class IstanbulCoverageProvider implements CoverageProvider {
   coverages: any[] = []
 
   initialize(ctx: Vitest) {
+    const config: CoverageIstanbulOptions = ctx.config.coverage
+
     this.ctx = ctx
-    this.options = resolveIstanbulOptions(ctx.config.coverage, ctx.config.root)
+    this.options = {
+      ...coverageConfigDefaults,
+
+      // User's options
+      ...config,
+
+      // Resolved fields
+      provider: 'istanbul',
+      reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory),
+      reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter),
+    }
 
     this.instrumenter = createInstrumenter({
       produceSourceMap: true,
@@ -141,6 +154,19 @@ export class IstanbulCoverageProvider implements CoverageProvider {
         statements: this.options.statements,
       })
     }
+
+    if (this.options.thresholdAutoUpdate && allTestsRun) {
+      this.updateThresholds({
+        coverageMap,
+        thresholds: {
+          branches: this.options.branches,
+          functions: this.options.functions,
+          lines: this.options.lines,
+          statements: this.options.statements,
+        },
+        configurationFile: this.ctx.server.config.configFile,
+      })
+    }
   }
 
   checkThresholds(coverageMap: CoverageMap, thresholds: Record<Threshold, number | undefined>) {
@@ -220,24 +246,6 @@ export class IstanbulCoverageProvider implements CoverageProvider {
   }
 }
 
-function resolveIstanbulOptions(options: CoverageIstanbulOptions, root: string): Options {
-  const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory)
-
-  const resolved: Options = {
-    ...coverageConfigDefaults,
-
-    // User's options
-    ...options,
-
-    // Resolved fields
-    provider: 'istanbul',
-    reportsDirectory,
-    reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter),
-  }
-
-  return resolved
-}
-
 /**
  * Remove possible query parameters from filenames
  * - From `/src/components/Header.component.ts?vue&type=script&src=true&lang.ts`
@@ -287,24 +295,3 @@ function isEmptyCoverageRange(range: libCoverage.Range) {
     || range.end.column === undefined
   )
 }
-
-function resolveReporters(configReporters: NonNullable<CoverageIstanbulOptions['reporter']>): Options['reporter'] {
-  // E.g. { reporter: "html" }
-  if (!Array.isArray(configReporters))
-    return [[configReporters, {}]]
-
-  const resolvedReporters: Options['reporter'] = []
-
-  for (const reporter of configReporters) {
-    if (Array.isArray(reporter)) {
-      // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
-      resolvedReporters.push([reporter[0], reporter[1] || {}])
-    }
-    else {
-      // E.g. { reporter: ["html", "json"]}
-      resolvedReporters.push([reporter, {}])
-    }
-  }
-
-  return resolvedReporters
-}

packages/vitest/coverage.d.ts

@@ -0,0 +1 @@
+export * from './dist/coverage.js'

packages/vitest/package.json

@@ -65,6 +65,10 @@
       "types": "./config.d.ts",
       "require": "./dist/config.cjs",
       "import": "./dist/config.js"
+    },
+    "./coverage": {
+      "types": "./coverage.d.ts",
+      "import": "./dist/coverage.js"
     }
   },
   "main": "./dist/index.js",
@@ -144,6 +148,7 @@
     "@edge-runtime/vm": "2.0.2",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/diff": "^5.0.2",
+    "@types/istanbul-lib-coverage": "^2.0.4",
     "@types/istanbul-reports": "^3.0.1",
     "@types/jsdom": "^21.1.0",
     "@types/micromatch": "^4.0.2",

packages/vitest/rollup.config.js

@@ -26,6 +26,7 @@ const entries = [
   'src/runtime/loader.ts',
   'src/runtime/entry.ts',
   'src/integrations/spy.ts',
+  'src/coverage.ts',
 ]
 
 const dtsEntries = [
@@ -36,6 +37,7 @@ const dtsEntries = [
   'src/runners.ts',
   'src/suite.ts',
   'src/config.ts',
+  'src/coverage.ts',
 ]
 
 const external = [

packages/vitest/src/coverage.ts

@@ -0,0 +1 @@
+export { BaseCoverageProvider } from './utils/coverage'

packages/vitest/src/types/coverage.ts

@@ -186,6 +186,13 @@ export interface BaseCoverageOptions {
    * @default undefined
    */
   statements?: number
+
+  /**
+   * Update threshold values automatically when current coverage is higher than earlier thresholds
+   *
+   * @default false
+   */
+  thresholdAutoUpdate?: boolean
 }
 
 export interface CoverageIstanbulOptions extends BaseCoverageOptions {

packages/vitest/src/utils/coverage.ts

@@ -0,0 +1,82 @@
+import { readFileSync, writeFileSync } from 'node:fs'
+import type { CoverageMap } from 'istanbul-lib-coverage'
+import type { BaseCoverageOptions, ResolvedCoverageOptions } from '../types'
+
+type Threshold = 'lines' | 'functions' | 'statements' | 'branches'
+
+const THRESHOLD_KEYS: Readonly<Threshold[]> = ['lines', 'functions', 'statements', 'branches']
+
+export class BaseCoverageProvider {
+  /**
+   * Check if current coverage is above configured thresholds and bump the thresholds if needed
+   */
+  updateThresholds({ configurationFile, coverageMap, thresholds }: {
+    coverageMap: CoverageMap
+    thresholds: Record<Threshold, number | undefined>
+    configurationFile?: string
+  }) {
+    // Thresholds cannot be updated if there is no configuration file and
+    // feature was enabled by CLI, e.g. --coverage.thresholdAutoUpdate
+    if (!configurationFile)
+      throw new Error('Missing configurationFile. The "coverage.thresholdAutoUpdate" can only be enabled when configuration file is used.')
+
+    const summary = coverageMap.getCoverageSummary()
+    const thresholdsToUpdate: Threshold[] = []
+
+    for (const key of THRESHOLD_KEYS) {
+      const threshold = thresholds[key] || 100
+      const actual = summary[key].pct
+
+      if (actual > threshold)
+        thresholdsToUpdate.push(key)
+    }
+
+    if (thresholdsToUpdate.length === 0)
+      return
+
+    const originalConfig = readFileSync(configurationFile, 'utf8')
+    let updatedConfig = originalConfig
+
+    for (const threshold of thresholdsToUpdate) {
+      // Find the exact match from the configuration file and replace the value
+      const previousThreshold = (thresholds[threshold] || 100).toString()
+      const pattern = new RegExp(`(${threshold}\\s*:\\s*)${previousThreshold.replace('.', '\\.')}`)
+      const matches = originalConfig.match(pattern)
+
+      if (matches)
+        updatedConfig = updatedConfig.replace(matches[0], matches[1] + summary[threshold].pct)
+      else
+        console.error(`Unable to update coverage threshold ${threshold}. No threshold found using pattern ${pattern}`)
+    }
+
+    if (updatedConfig !== originalConfig) {
+      // eslint-disable-next-line no-console
+      console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.')
+      writeFileSync(configurationFile, updatedConfig, 'utf-8')
+    }
+  }
+
+  /**
+   * Resolve reporters from various configuration options
+   */
+  resolveReporters(configReporters: NonNullable<BaseCoverageOptions['reporter']>): ResolvedCoverageOptions['reporter'] {
+    // E.g. { reporter: "html" }
+    if (!Array.isArray(configReporters))
+      return [[configReporters, {}]]
+
+    const resolvedReporters: ResolvedCoverageOptions['reporter'] = []
+
+    for (const reporter of configReporters) {
+      if (Array.isArray(reporter)) {
+        // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
+        resolvedReporters.push([reporter[0], reporter[1] || {}])
+      }
+      else {
+        // E.g. { reporter: ["html", "json"]}
+        resolvedReporters.push([reporter, {}])
+      }
+    }
+
+    return resolvedReporters
+  }
+}