fix(coverage): custom providers to work inside worker threads (#2817)

docs/config/index.md

@@ -617,7 +617,7 @@ Isolate environment for each test file. Does not work if you disable [`--threads
 
 ### coverage
 
-You can use [`c8`](https://github.com/bcoe/c8) or [`istanbul`](https://istanbul.js.org/) for coverage collection.
+You can use [`c8`](https://github.com/bcoe/c8), [`istanbul`](https://istanbul.js.org/)  or [a custom coverage solution](/guide/coverage#custom-coverage-provider) for coverage collection.
 
 You can provide coverage options to CLI with dot notation:
 
@@ -631,7 +631,7 @@ If you are using coverage options with dot notation, don't forget to specify `--
 
 #### provider
 
-- **Type:** `'c8' | 'istanbul'`
+- **Type:** `'c8' | 'istanbul' | 'custom'`
 - **Default:** `'c8'`
 - **CLI:** `--coverage.provider=<provider>`
 
@@ -863,6 +863,14 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#ignoring-methods)
 
 Watermarks for statements, lines, branches and functions. See [istanbul documentation](https://github.com/istanbuljs/nyc#high-and-low-watermarks) for more information.
 
+#### customProviderModule
+
+- **Type:** `string`
+- **Available for providers:** `'custom'`
+- **CLI:** `--coverage.customProviderModule=<path or module name>`
+
+Specifies the module name or path for the custom coverage provider module. See [Guide - Custom Coverage Provider](/guide/coverage#custom-coverage-provider) for more information.
+
 ### testNamePattern
 
 - **Type** `string | RegExp`

docs/guide/coverage.md

@@ -71,22 +71,50 @@ export default defineConfig({
 
 ## Custom Coverage Provider
 
-It's also possible to provide your custom coverage provider by passing an object to the `test.coverage.provider`:
+It's also possible to provide your custom coverage provider by passing `'custom'` in `test.coverage.provider`:
 
 ```ts
 // vite.config.ts
 import { defineConfig } from 'vitest/config'
-import CustomCoverageProvider from 'my-custom-coverage-provider'
 
 export default defineConfig({
   test: {
     coverage: {
-      provider: CustomCoverageProvider()
+      provider: 'custom',
+      customProviderModule: 'my-custom-coverage-provider'
     },
   },
 })
 ```
 
+The custom providers require a `customProviderModule` option which is a module name or path where to load the `CoverageProviderModule` from. It must export an object that implements `CoverageProviderModule` as default export:
+
+```ts
+// my-custom-coverage-provider.ts
+import type { CoverageProvider, CoverageProviderModule, ResolvedCoverageOptions, Vitest } from 'vitest'
+
+const CustomCoverageProviderModule: CoverageProviderModule = {
+  getProvider(): CoverageProvider {
+    return new CustomCoverageProvider()
+  },
+
+  // Implements rest of the CoverageProviderModule ...
+}
+
+class CustomCoverageProvider implements CoverageProvider {
+  name = 'custom-coverage-provider'
+  options!: ResolvedCoverageOptions
+
+  initialize(ctx: Vitest) {
+    this.options = ctx.config.coverage
+  }
+
+  // Implements rest of the CoverageProvider ...
+}
+
+export default CustomCoverageProviderModule
+```
+
 Please refer to the type definition for more details.
 
 ## Changing the default coverage folder location

packages/vitest/src/integrations/coverage.ts

@@ -1,34 +1,47 @@
 import { importModule } from 'local-pkg'
 import type { CoverageOptions, CoverageProvider, CoverageProviderModule } from '../types'
 
-export const CoverageProviderMap = {
+interface Loader {
+  executeId: (id: string) => Promise<{ default: CoverageProviderModule }>
+}
+
+export const CoverageProviderMap: Record<string, string> = {
   c8: '@vitest/coverage-c8',
   istanbul: '@vitest/coverage-istanbul',
 }
 
-export async function resolveCoverageProvider(provider: NonNullable<CoverageOptions['provider']>) {
-  if (typeof provider === 'string') {
-    const pkg = CoverageProviderMap[provider]
-    if (!pkg)
-      throw new Error(`Unknown coverage provider: ${provider}`)
-    return await importModule<CoverageProviderModule>(pkg)
+async function resolveCoverageProviderModule(options: CoverageOptions & Required<Pick<CoverageOptions, 'provider'>>, loader: Loader) {
+  const provider = options.provider
+
+  if (provider === 'c8' || provider === 'istanbul')
+    return await importModule<CoverageProviderModule>(CoverageProviderMap[provider])
+
+  let customProviderModule
+
+  try {
+    customProviderModule = await loader.executeId(options.customProviderModule)
   }
-  else {
-    return provider
+  catch (error) {
+    throw new Error(`Failed to load custom CoverageProviderModule from ${options.customProviderModule}`, { cause: error })
   }
+
+  if (customProviderModule.default == null)
+    throw new Error(`Custom CoverageProviderModule loaded from ${options.customProviderModule} was not the default export`)
+
+  return customProviderModule.default
 }
 
-export async function getCoverageProvider(options?: CoverageOptions): Promise<CoverageProvider | null> {
-  if (options?.enabled && options?.provider) {
-    const { getProvider } = await resolveCoverageProvider(options.provider)
+export async function getCoverageProvider(options: CoverageOptions, loader: Loader): Promise<CoverageProvider | null> {
+  if (options.enabled && options.provider) {
+    const { getProvider } = await resolveCoverageProviderModule(options, loader)
     return await getProvider()
   }
   return null
 }
 
-export async function takeCoverageInsideWorker(options: CoverageOptions) {
+export async function takeCoverageInsideWorker(options: CoverageOptions, loader: Loader) {
   if (options.enabled && options.provider) {
-    const { takeCoverage } = await resolveCoverageProvider(options.provider)
+    const { takeCoverage } = await resolveCoverageProviderModule(options, loader)
     return await takeCoverage?.()
   }
 }

packages/vitest/src/node/cli-api.ts

@@ -50,9 +50,9 @@ export async function startVitest(
 
   if (mode === 'test' && ctx.config.coverage.enabled) {
     const provider = ctx.config.coverage.provider || 'c8'
-    if (typeof provider === 'string') {
-      const requiredPackages = CoverageProviderMap[provider]
+    const requiredPackages = CoverageProviderMap[provider]
 
+    if (requiredPackages) {
       if (!await ensurePackageInstalled(requiredPackages, root)) {
         process.exitCode = 1
         return ctx

packages/vitest/src/node/core.ts

@@ -124,7 +124,7 @@ export class Vitest {
   async initCoverageProvider() {
     if (this.coverageProvider !== undefined)
       return
-    this.coverageProvider = await getCoverageProvider(this.config.coverage)
+    this.coverageProvider = await getCoverageProvider(this.config.coverage, this.runner)
     if (this.coverageProvider) {
       await this.coverageProvider.initialize(this)
       this.config.coverage = this.coverageProvider.resolveOptions()

packages/vitest/src/runtime/entry.ts

@@ -68,7 +68,7 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor):
 
   const originalOnAfterRun = testRunner.onAfterRun
   testRunner.onAfterRun = async (files) => {
-    const coverage = await takeCoverageInsideWorker(config.coverage)
+    const coverage = await takeCoverageInsideWorker(config.coverage, executor)
     rpc().onAfterSuiteRun({ coverage })
     await originalOnAfterRun?.call(testRunner, files)
   }

packages/vitest/src/types/coverage.ts

@@ -53,12 +53,13 @@ export type CoverageReporter =
   | 'text-summary'
   | 'text'
 
-type Provider = 'c8' | 'istanbul' | CoverageProviderModule | undefined
+type Provider = 'c8' | 'istanbul' | 'custom' | undefined
 
 export type CoverageOptions<T extends Provider = Provider> =
-  T extends CoverageProviderModule ? ({ provider: T } & BaseCoverageOptions) :
-    T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
-        ({ provider?: T } & CoverageC8Options)
+  T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
+    T extends 'c8' ? ({ provider: T } & CoverageC8Options) :
+      T extends 'custom' ? ({ provider: T } & CustomProviderOptions) :
+          ({ provider?: T } & (CoverageC8Options))
 
 /** Fields that have default values. Internally these will always be defined. */
 type FieldsWithDefaultValues =
@@ -233,3 +234,8 @@ export interface CoverageC8Options extends BaseCoverageOptions {
    */
   100?: boolean
 }
+
+export interface CustomProviderOptions extends Pick<BaseCoverageOptions, FieldsWithDefaultValues> {
+  /** Name of the module or path to a file to load the custom provider from */
+  customProviderModule: string
+}

test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap

@@ -0,0 +1,26 @@
+// Vitest Snapshot v1
+
+exports[`custom json report 1`] = `
+{
+  "calls": [
+    "initialized with context",
+    "resolveOptions",
+    "clean with force",
+    "onBeforeFilesRun",
+    "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
+    "reportCoverage with {\\"allTestsRun\\":true}",
+  ],
+  "transformedFiles": [
+    "<process-cwd>/src/Counter/Counter.component.ts",
+    "<process-cwd>/src/Counter/Counter.vue",
+    "<process-cwd>/src/Counter/index.ts",
+    "<process-cwd>/src/Defined.vue",
+    "<process-cwd>/src/Hello.vue",
+    "<process-cwd>/src/another-setup.ts",
+    "<process-cwd>/src/implicitElse.ts",
+    "<process-cwd>/src/importEnv.ts",
+    "<process-cwd>/src/index.mts",
+    "<process-cwd>/src/utils.ts",
+  ],
+}
+`;

test/coverage-test/coverage-report-tests/custom.report.test.ts

@@ -0,0 +1,12 @@
+/*
+ * Custom coverage provider specific test cases
+ */
+
+import { readFileSync } from 'fs'
+import { expect, test } from 'vitest'
+
+test('custom json report', async () => {
+  const report = readFileSync('./coverage/custom-coverage-provider-report.json', 'utf-8')
+
+  expect(JSON.parse(report)).toMatchSnapshot()
+})

test/coverage-test/coverage-report-tests/utils.ts

@@ -1,3 +1,4 @@
+import { readFileSync } from 'fs'
 import { normalize } from 'pathe'
 
 interface CoverageFinalJson {
@@ -17,8 +18,7 @@ interface CoverageFinalJson {
  * Normalizes paths to keep contents consistent between OS's
  */
 export async function readCoverageJson() {
-  // @ts-expect-error -- generated file
-  const { default: jsonReport } = await import('./coverage/coverage-final.json') as CoverageFinalJson
+  const jsonReport = JSON.parse(readFileSync('./coverage/coverage-final.json', 'utf8')) as CoverageFinalJson
 
   const normalizedReport: CoverageFinalJson['default'] = {}
 
@@ -30,6 +30,6 @@ export async function readCoverageJson() {
   return normalizedReport
 }
 
-function normalizeFilename(filename: string) {
+export function normalizeFilename(filename: string) {
   return normalize(filename).replace(normalize(process.cwd()), '<process-cwd>')
 }

test/coverage-test/custom-provider.ts

@@ -0,0 +1,75 @@
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'
+import type { AfterSuiteRunMeta, CoverageProvider, CoverageProviderModule, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
+
+import { normalizeFilename } from './coverage-report-tests/utils'
+
+const CustomCoverageProviderModule: CoverageProviderModule = {
+  getProvider(): CoverageProvider {
+    return new CustomCoverageProvider()
+  },
+
+  takeCoverage() {
+    return { customCoverage: 'Coverage report passed from workers to main thread' }
+  },
+}
+
+/**
+ * Provider that simply keeps track of the functions that were called
+ */
+class CustomCoverageProvider implements CoverageProvider {
+  name = 'custom-coverage-provider'
+
+  options!: ResolvedCoverageOptions
+  calls: Set<string> = new Set()
+  transformedFiles: Set<string> = new Set()
+
+  initialize(ctx: Vitest) {
+    this.options = ctx.config.coverage
+
+    this.calls.add(`initialized ${ctx ? 'with' : 'without'} context`)
+  }
+
+  clean(force: boolean) {
+    this.calls.add(`clean ${force ? 'with' : 'without'} force`)
+  }
+
+  onBeforeFilesRun() {
+    this.calls.add('onBeforeFilesRun')
+  }
+
+  onAfterSuiteRun(meta: AfterSuiteRunMeta) {
+    this.calls.add(`onAfterSuiteRun with ${JSON.stringify(meta)}`)
+  }
+
+  reportCoverage(reportContext?: ReportContext) {
+    this.calls.add(`reportCoverage with ${JSON.stringify(reportContext)}`)
+
+    const jsonReport = JSON.stringify({
+      calls: Array.from(this.calls.values()),
+      transformedFiles: Array.from(this.transformedFiles.values()).sort(),
+    }, null, 2)
+
+    if (existsSync('./coverage'))
+      rmSync('./coverage', { maxRetries: 10, recursive: true })
+
+    mkdirSync('./coverage')
+    writeFileSync('./coverage/custom-coverage-provider-report.json', jsonReport, 'utf-8')
+  }
+
+  onFileTransform(code: string, id: string) {
+    const filename = normalizeFilename(id).split('?')[0]
+
+    if (/\/src\//.test(filename))
+      this.transformedFiles.add(filename)
+
+    return { code }
+  }
+
+  resolveOptions(): ResolvedCoverageOptions {
+    this.calls.add('resolveOptions')
+
+    return this.options
+  }
+}
+
+export default CustomCoverageProviderModule

test/coverage-test/package.json

@@ -2,10 +2,11 @@
   "name": "@vitest/test-coverage",
   "private": true,
   "scripts": {
-    "test": "pnpm run test:c8 && pnpm run test:istanbul && pnpm run test:types",
+    "test": "pnpm test:c8 && pnpm test:istanbul && pnpm test:custom && pnpm test:types",
     "test:c8": "node ./testing.mjs --provider c8",
+    "test:custom": "node ./testing.mjs --provider custom",
     "test:istanbul": "node ./testing.mjs --provider istanbul",
-    "test:types": "vitest typecheck --run"
+    "test:types": "vitest typecheck --run --reporter verbose"
   },
   "devDependencies": {
     "@vitejs/plugin-vue": "latest",