feat: support update: "none" and add docs about snapshots behavior on CI

docs/config/update.md

@@ -5,11 +5,17 @@ outline: deep
 
 # update <CRoot /> {#update}
 
-- **Type:** `boolean | 'new' | 'all'`
+- **Type:** `boolean | 'new' | 'all' | 'none'`
 - **Default:** `false`
-- **CLI:** `-u`, `--update`, `--update=false`, `--update=new`
+- **CLI:** `-u`, `--update`, `--update=false`, `--update=new`, `--update=none`
 
-Update snapshot files. The behaviour depends on the value:
+Define snapshot update behavior.
 
-- `true` or `'all'`: updates all changed snapshots and delete obsolete ones
+- `true` or `'all'`: updates all changed snapshots and deletes obsolete ones
 - `new`: generates new snapshots without changing or deleting obsolete ones
+- `none`: does not write snapshots and fails on snapshot mismatches, missing snapshots, and obsolete snapshots
+
+When `update` is `false` (the default), Vitest resolves snapshot update mode by environment:
+
+- Local runs (non-CI): works same as `new`
+- CI runs (`process.env.CI` is truthy): works same as `none`

docs/guide/cli-generated.md

@@ -16,7 +16,7 @@ Path to config file
 - **CLI:** `-u, --update [type]`
 - **Config:** [update](/config/update)
 
-Update snapshot (accepts boolean, "new" or "all")
+Update snapshot (accepts boolean, "new", "all" or "none")
 
 ### watch
 

docs/guide/snapshot.md

@@ -79,6 +79,12 @@ Or you can use the `--update` or `-u` flag in the CLI to make Vitest update snap
 vitest -u
 ```
 
+### CI behavior
+
+By default, Vitest does not write snapshots in CI (`process.env.CI` is truthy) and any snapshot mismatches, missing snapshots, and obsolete snapshots fail the run. See [`update`](/config/update) for the details.
+
+An **obsolete snapshot** is a snapshot entry (or snapshot file) that no longer matches any collected test. This usually happens after removing or renaming tests.
+
 ## File Snapshots
 
 When calling `toMatchSnapshot()`, we store all snapshots in a formatted snap file. That means we need to escape some characters (namely the double-quote `"` and backtick `` ` ``) in the snapshot string. Meanwhile, you might lose the syntax highlighting for the snapshot content (if they are in some language).

packages/vitest/src/node/cli/cli-config.ts

@@ -84,7 +84,7 @@ export const cliOptionsConfig: VitestCLIOptions = {
   },
   update: {
     shorthand: 'u',
-    description: 'Update snapshot (accepts boolean, "new" or "all")',
+    description: 'Update snapshot (accepts boolean, "new", "all" or "none")',
     argument: '[type]',
   },
   watch: {

packages/vitest/src/node/config/resolveConfig.ts

@@ -566,7 +566,7 @@ export function resolveConfig(
     expand: resolved.expandSnapshotDiff ?? false,
     snapshotFormat: resolved.snapshotFormat || {},
     updateSnapshot:
-      UPDATE_SNAPSHOT === 'all' || UPDATE_SNAPSHOT === 'new'
+      UPDATE_SNAPSHOT === 'all' || UPDATE_SNAPSHOT === 'new' || UPDATE_SNAPSHOT === 'none'
         ? UPDATE_SNAPSHOT
         : isCI && !UPDATE_SNAPSHOT ? 'none' : UPDATE_SNAPSHOT ? 'all' : 'new',
     resolveSnapshotPath: options.resolveSnapshotPath,

packages/vitest/src/node/types/config.ts

@@ -380,7 +380,7 @@ export interface InlineConfig {
    *
    * @default false
    */
-  update?: boolean | 'all' | 'new'
+  update?: boolean | 'all' | 'new' | 'none'
 
   /**
    * Watch mode

test/cli/test/around-each.test.ts

@@ -1463,6 +1463,9 @@ test('aroundAll throws error when runSuite is not called', async () => {
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "no-run.test.ts": {
+        "__module_errors__": [
+          "The \`runSuite()\` callback was not called in the \`aroundAll\` hook. Make sure to call \`runSuite()\` to run the suite.",
+        ],
         "test": "skipped",
       },
     }
@@ -1584,6 +1587,9 @@ test('aroundAll setup phase timeout', async () => {
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "timeout.test.ts": {
+        "__module_errors__": [
+          "The setup phase of "aroundAll" hook timed out after 10ms.",
+        ],
         "test": "skipped",
       },
     }
@@ -1635,6 +1641,9 @@ test('aroundAll teardown phase timeout', async () => {
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "teardown-timeout.test.ts": {
+        "__module_errors__": [
+          "The teardown phase of "aroundAll" hook timed out after 10ms.",
+        ],
         "test": "passed",
       },
     }
@@ -1974,6 +1983,9 @@ test('tests are skipped when aroundAll setup fails', async () => {
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "aroundAll-setup-error.test.ts": {
+        "__module_errors__": [
+          "aroundAll setup error",
+        ],
         "test should be skipped": "skipped",
       },
     }
@@ -2461,6 +2473,9 @@ test('nested aroundAll setup error is not propagated to outer runSuite catch', a
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "nested-around-all-setup-error.test.ts": {
+        "__module_errors__": [
+          "inner aroundAll setup error",
+        ],
         "repro": "skipped",
       },
     }
@@ -2524,6 +2539,9 @@ test('nested aroundAll teardown error is not propagated to outer runSuite catch'
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "nested-around-all-teardown-error.test.ts": {
+        "__module_errors__": [
+          "inner aroundAll teardown error",
+        ],
         "repro": "passed",
       },
     }
@@ -2712,6 +2730,11 @@ test('three nested aroundAll teardown errors are all reported', async () => {
   expect(errorTree()).toMatchInlineSnapshot(`
     {
       "triple-around-all-teardown-errors.test.ts": {
+        "__module_errors__": [
+          "inner aroundAll teardown error",
+          "middle aroundAll teardown error",
+          "outer aroundAll teardown error",
+        ],
         "repro": "passed",
       },
     }

test/cli/test/mocking.test.ts

@@ -43,26 +43,14 @@ test('spy is not called here', () => {
 })
 
 test('invalid packages', async () => {
-  const { results, errorTree } = await runVitest({
+  const { stderr, errorTree } = await runVitest({
     root: path.join(import.meta.dirname, '../fixtures/invalid-package'),
   })
-  const testModuleErrors = Object.fromEntries(
-    results.map(testModule => [
-      testModule.relativeModuleId,
-      testModule.errors().map(e => e.message),
-    ]),
-  )
 
   // requires Vite 8 for relaxed import analysis validataion
   // https://github.com/vitejs/vite/pull/21601
   if (rolldownVersion) {
-    expect(testModuleErrors).toMatchInlineSnapshot(`
-      {
-        "mock-bad-dep.test.ts": [],
-        "mock-wrapper-and-bad-dep.test.ts": [],
-        "mock-wrapper.test.ts": [],
-      }
-    `)
+    expect(stderr).toMatchInlineSnapshot(`""`)
     expect(errorTree()).toMatchInlineSnapshot(`
       {
         "mock-bad-dep.test.ts": {
@@ -78,32 +66,31 @@ test('invalid packages', async () => {
     `)
   }
   else {
-    expect(testModuleErrors).toMatchInlineSnapshot(`
+    expect(errorTree()).toMatchInlineSnapshot(`
       {
-        "mock-bad-dep.test.ts": [
-          "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
-        ],
-        "mock-wrapper-and-bad-dep.test.ts": [
-          "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
-        ],
-        "mock-wrapper.test.ts": [
-          "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
-        ],
+        "mock-bad-dep.test.ts": {
+          "__module_errors__": [
+            "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
+          ],
+        },
+        "mock-wrapper-and-bad-dep.test.ts": {
+          "__module_errors__": [
+            "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
+          ],
+        },
+        "mock-wrapper.test.ts": {
+          "__module_errors__": [
+            "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.",
+          ],
+        },
       }
     `)
-    expect(errorTree()).toMatchInlineSnapshot(`
-        {
-          "mock-bad-dep.test.ts": {},
-          "mock-wrapper-and-bad-dep.test.ts": {},
-          "mock-wrapper.test.ts": {},
-        }
-      `)
   }
 })
 
 test('mocking modules with syntax error', async () => {
   // TODO: manual mocked module still gets transformed so this is not supported yet.
-  const { errorTree, results } = await runInlineTests({
+  const { errorTree } = await runInlineTests({
     './syntax-error.js': `syntax error`,
     './basic.test.js': /* ts */ `
 import * as dep from './syntax-error.js'
@@ -118,38 +105,31 @@ test('can mock invalid module', () => {
     `,
   })
 
-  const testModuleErrors = Object.fromEntries(
-    results.map(testModule => [
-      testModule.relativeModuleId,
-      testModule.errors().map(e => e.message),
-    ]),
-  )
   if (rolldownVersion) {
-    expect(testModuleErrors).toMatchInlineSnapshot(`
+    expect(errorTree()).toMatchInlineSnapshot(`
       {
-        "basic.test.js": [
-          "Parse failure: Parse failed with 1 error:
+        "basic.test.js": {
+          "__module_errors__": [
+            "Parse failure: Parse failed with 1 error:
       Expected a semicolon or an implicit semicolon after a statement, but found none
       1: syntax error
                ^
       At file: /syntax-error.js:1:6",
-        ],
+          ],
+        },
       }
     `)
   }
   else {
-    expect(testModuleErrors).toMatchInlineSnapshot(`
+    expect(errorTree()).toMatchInlineSnapshot(`
       {
-        "basic.test.js": [
-          "Parse failure: Expected ';', '}' or <eof>
+        "basic.test.js": {
+          "__module_errors__": [
+            "Parse failure: Expected ';', '}' or <eof>
       At file: /syntax-error.js:1:7",
-        ],
+          ],
+        },
       }
     `)
   }
-  expect(errorTree()).toMatchInlineSnapshot(`
-    {
-      "basic.test.js": {},
-    }
-  `)
 })

test/snapshots/test/ci.test.ts

@@ -0,0 +1,54 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { expect, test } from 'vitest'
+import { runVitestCli } from '../../test-utils'
+
+test('CI behavior', async () => {
+  // cleanup snapshot
+  const root = path.join(import.meta.dirname, 'fixtures/ci')
+  fs.rmSync(path.join(root, '__snapshots__'), { recursive: true, force: true })
+
+  // snapshot fails with CI
+  let result = await runVitestCli({
+    nodeOptions: {
+      env: {
+        CI: 'true',
+        GITHUB_ACTIONS: 'true',
+      },
+    },
+  }, '--root', root)
+  expect(result.stderr).toMatchInlineSnapshot(`
+    "
+    ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯
+
+     FAIL  basic.test.ts > basic
+    Error: Snapshot \`basic 1\` mismatched
+     ❯ basic.test.ts:4:16
+          2|
+          3| test("basic", () => {
+          4|   expect("ok").toMatchSnapshot()
+           |                ^
+          5| })
+          6|
+
+    ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯
+
+    "
+  `)
+
+  // snapshot created without CI
+  result = await runVitestCli(
+    {
+      nodeOptions: {
+        env: {
+          CI: '',
+          GITHUB_ACTIONS: '',
+        },
+      },
+    },
+    '--root',
+    root,
+  )
+  expect(result.stderr).toMatchInlineSnapshot(`""`)
+  expect(result.stdout).toContain('Snapshots  1 written')
+})

test/snapshots/test/fixtures/ci/.gitignore

@@ -0,0 +1 @@
+__snapshots__

test/snapshots/test/fixtures/ci/basic.test.ts

@@ -0,0 +1,5 @@
+import { test, expect } from "vitest"
+
+test("basic", () => {
+  expect("ok").toMatchSnapshot()
+})