fix(browser): take failure screenshot if toMatchScreenshot can't capture a stable screenshot

packages/browser/src/client/tester/expect/toMatchScreenshot.ts

@@ -120,5 +120,8 @@ export default async function toMatchScreenshot(
           ]
             .filter(element => element !== null)
             .join('\n'),
+    meta: {
+      outcome: result.outcome,
+    },
   }
 }

packages/browser/src/client/tester/runner.ts

@@ -156,14 +156,16 @@ export function createBrowserRunner(
     }
 
     onTaskFinished = async (task: Task) => {
+      const lastErrorContext = task.result?.errors?.at(-1)?.context
       if (
         this.config.browser.screenshotFailures
         && document.body.clientHeight > 0
         && task.result?.state === 'fail'
         && task.type === 'test'
-        && task.artifacts.every(
-          artifact => artifact.type !== 'internal:toMatchScreenshot',
-        )
+        && !(
+          lastErrorContext
+          && Reflect.get(lastErrorContext, 'assertionName') === 'toMatchScreenshot'
+          && Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot')
       ) {
         const screenshot = await page.screenshot({
           timeout: this.config.browser.providerOptions?.actionTimeout ?? 5_000,

packages/browser/src/node/commands/screenshotMatcher/index.ts

@@ -273,6 +273,7 @@ function buildOutput(
     case 'unstable-screenshot':
       return {
         pass: false,
+        outcome: outcome.type,
         reference: outcome.reference && {
           path: outcome.reference.path,
           width: outcome.reference.image.metadata.width,
@@ -286,6 +287,7 @@ function buildOutput(
     case 'missing-reference': {
       return {
         pass: false,
+        outcome: outcome.type,
         reference: {
           path: outcome.reference.path,
           width: outcome.reference.image.metadata.width,
@@ -302,11 +304,12 @@ function buildOutput(
     case 'update-reference':
     case 'matched-immediately':
     case 'matched-after-comparison':
-      return { pass: true }
+      return { pass: true, outcome: outcome.type }
 
     case 'mismatch':
       return {
         pass: false,
+        outcome: outcome.type,
         reference: {
           path: outcome.reference.path,
           width: outcome.reference.image.metadata.width,
@@ -333,6 +336,7 @@ function buildOutput(
 
       return {
         pass: false,
+        outcome: null as never,
         actual: null,
         reference: null,
         diff: null,

packages/browser/src/shared/screenshotMatcher/types.ts

@@ -17,12 +17,20 @@ interface ScreenshotData { path: string; width: number; height: number }
 export type ScreenshotMatcherOutput = Promise<
   {
     pass: false
+    outcome:
+      | 'unstable-screenshot'
+      | 'missing-reference'
+      | 'mismatch'
     reference: ScreenshotData | null
     actual: ScreenshotData | null
     diff: ScreenshotData | null
     message: string
   }
   | {
     pass: true
+    outcome:
+      | 'update-reference'
+      | 'matched-immediately'
+      | 'matched-after-comparison'
   }
 >

packages/expect/src/jest-extend.ts

@@ -65,7 +65,12 @@ function getMatcherState(
 }
 
 class JestExtendError extends Error {
-  constructor(message: string, public actual?: any, public expected?: any) {
+  constructor(
+    message: string,
+    public actual?: any,
+    public expected?: any,
+    public context?: { assertionName: string; meta?: object },
+  ) {
     super(message)
   }
 }
@@ -92,23 +97,33 @@ function JestExtendPlugin(
             && typeof (result as any).then === 'function'
           ) {
             const thenable = result as PromiseLike<SyncExpectationResult>
-            return thenable.then(({ pass, message, actual, expected }) => {
+            return thenable.then(({ pass, message, actual, expected, meta }) => {
               if ((pass && isNot) || (!pass && !isNot)) {
                 const errorMessage = customMessage != null
                   ? customMessage
                   : message()
-                throw new JestExtendError(errorMessage, actual, expected)
+                throw new JestExtendError(
+                  errorMessage,
+                  actual,
+                  expected,
+                  { assertionName: expectAssertionName, meta },
+                )
               }
             })
           }
 
-          const { pass, message, actual, expected } = result as SyncExpectationResult
+          const { pass, message, actual, expected, meta } = result as SyncExpectationResult
 
           if ((pass && isNot) || (!pass && !isNot)) {
             const errorMessage = customMessage != null
               ? customMessage
               : message()
-            throw new JestExtendError(errorMessage, actual, expected)
+            throw new JestExtendError(
+              errorMessage,
+              actual,
+              expected,
+              { assertionName: expectAssertionName, meta },
+            )
           }
         }
 

packages/expect/src/types.ts

@@ -91,6 +91,7 @@ export interface SyncExpectationResult {
   message: () => string
   actual?: any
   expected?: any
+  meta?: object
 }
 
 export type AsyncExpectationResult = Promise<SyncExpectationResult>

test/browser/specs/failure-screenshot.test.ts

@@ -0,0 +1,100 @@
+import type { TestFsStructure } from '../../test-utils'
+import { describe, expect, test } from 'vitest'
+import { runInlineTests } from '../../test-utils'
+import utilsContent from '../fixtures/expect-dom/utils?raw'
+import { instances, provider } from '../settings'
+
+const testFilename = 'basic.test.ts'
+
+async function runBrowserTests(
+  structure: TestFsStructure,
+) {
+  return runInlineTests({
+    ...structure,
+    'vitest.config.js': `
+      import { ${provider.name} } from '@vitest/browser-${provider.name}'
+      export default {
+        test: {
+          browser: {
+            enabled: true,
+            screenshotFailures: true,
+            provider: ${provider.name}(),
+            ui: false,
+            headless: true,
+            instances: ${JSON.stringify(instances.slice(0, 1) /* logic not bound to browser instance */)},
+          },
+          reporters: ['verbose'],
+          update: 'new',
+        },
+      }`,
+  })
+}
+
+describe('failure screenshots', () => {
+  describe('`toMatchScreenshot`', () => {
+    test('usually does NOT produce a failure screenshot', async () => {
+      const { stderr } = await runBrowserTests(
+        {
+          [testFilename]: /* ts */`
+            import { page } from 'vitest/browser'
+            import { test } from 'vitest'
+            import { render } from './utils'
+
+            test('screenshot-initial', async ({ expect }) => {
+              render('<div data-testid="el">Test</div>')
+              await expect(page.getByTestId('el')).toMatchScreenshot()
+            })
+          `,
+          'utils.ts': utilsContent,
+        },
+      )
+
+      expect(stderr).toContain('No existing reference screenshot found; a new one was created.')
+      expect(stderr).not.toContain('Failure screenshot:')
+    })
+
+    test('unstable screenshot fails produces a failure screenshot', async () => {
+      const { stderr } = await runBrowserTests(
+        {
+          [testFilename]: /* ts */`
+            import { page } from 'vitest/browser'
+            import { test } from 'vitest'
+            import { render } from './utils'
+
+            test('screenshot-unstable', async ({ expect }) => {
+              render('<div data-testid="el">Test</div>')
+              await expect(page.getByTestId('el')).toMatchScreenshot({ timeout: 1 })
+            })
+          `,
+          'utils.ts': utilsContent,
+        },
+      )
+
+      expect(stderr).toContain('Could not capture a stable screenshot within 1ms.')
+      expect(stderr).toContain('Failure screenshot:')
+    })
+
+    test('`expect.soft` produces a failure screenshot', async () => {
+      const { stderr } = await runBrowserTests(
+        {
+          [testFilename]: /* ts */`
+            import { page } from 'vitest/browser'
+            import { test } from 'vitest'
+            import { render } from './utils'
+
+            test('screenshot-soft-then-fail', async ({ expect }) => {
+              render('<div data-testid="el">Test</div>')
+              await expect.soft(page.getByTestId('el')).toMatchScreenshot()
+              expect(1).toBe(2)
+            })
+          `,
+          'utils.ts': utilsContent,
+        },
+      )
+
+      expect(stderr).toContain('No existing reference screenshot found; a new one was created.')
+      expect(stderr).toContain('expected 1 to be 2')
+      expect(stderr).toContain('Failure screenshot:')
+    })
+  })
+})