Add --ci to jest tests in CI (#60432)

## What?

Ensures snapshot tests fail instead of being written in CI.

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->


Closes NEXT-2035

run-tests.js

@@ -422,6 +422,7 @@ ${ENDGROUP}`)
         ...(shouldRecordTestWithReplay
           ? [`--config=jest.replay.config.js`]
           : []),
+        ...(process.env.CI ? ['--ci'] : []),
         '--runInBand',
         '--forceExit',
         '--verbose',

test/development/acceptance-app/ReactRefreshLogBox.test.ts

@@ -5,6 +5,8 @@ import { check, describeVariants as describe } from 'next-test-utils'
 import path from 'path'
 import { outdent } from 'outdent'
 
+const IS_TURBOPACK = Boolean(process.env.TURBOPACK)
+
 describe.each(['default', 'turbo'])('ReactRefreshLogBox app %s', () => {
   const { next } = nextTestSetup({
     files: new FileRef(path.join(__dirname, 'fixtures', 'default-template')),
@@ -220,7 +222,19 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox app %s', () => {
 
     const source = await session.getRedboxSource()
     expect(next.normalizeTestDirContent(source)).toMatchInlineSnapshot(
-      next.normalizeSnapshot(`
+      next.normalizeSnapshot(
+        IS_TURBOPACK
+          ? `
+      "./index.js:7:1
+      Parsing ecmascript source code failed
+        5 |     div
+        6 |   )
+      > 7 | }
+          |  ^
+
+      Unexpected eof"
+    `
+          : `
         "./index.js
         Error: 
           x Unexpected token. Did you mean \`{'}'}\` or \`}\`?
@@ -246,7 +260,8 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox app %s', () => {
         Import trace for requested module:
         ./index.js
         ./app/page.js"
-      `)
+      `
+      )
     )
 
     await cleanup()
@@ -344,13 +359,18 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox app %s', () => {
     await session.patch('index.module.css', `.button {`)
     expect(await session.hasRedbox(true)).toBe(true)
     const source = await session.getRedboxSource()
-    expect(source).toMatch('./index.module.css:1:1')
-    expect(source).toMatch('Syntax error: ')
-    expect(source).toMatch('Unclosed block')
+    expect(source).toMatch(
+      IS_TURBOPACK ? './index.module.css:1:8' : './index.module.css:1:1'
+    )
+    if (!IS_TURBOPACK) {
+      expect(source).toMatch('Syntax error: ')
+      expect(source).toMatch('Unclosed block')
+    }
     expect(source).toMatch('> 1 | .button {')
-    expect(source).toMatch('    | ^')
+    expect(source).toMatch(IS_TURBOPACK ? '    |         ^' : '    | ^')
 
-    // Not local error
+    // Checks for selectors that can't be prefixed.
+    // Selector "button" is not pure (pure selectors must contain at least one local class or id)
     await session.patch('index.module.css', `button {}`)
     expect(await session.hasRedbox(true)).toBe(true)
     const source2 = await session.getRedboxSource()

test/development/acceptance-app/__snapshots__/ReactRefreshLogBox.test.ts.snap

@@ -98,3 +98,77 @@ exports[`ReactRefreshLogBox app default should strip whitespace correctly with n
   10 |   click me
   11 | </a>"
 `;
+
+exports[`ReactRefreshLogBox app turbo Can't resolve @import in CSS file 1`] = `
+"./app/styles1.css:1:0
+Parsing css source code failed
+> 1 | @import "./styles2.css"
+    | ^^^^^^^^^^^^^^^^^^^^^^^
+
+Unexpected end of file, but expected ';' or '{'"
+`;
+
+exports[`ReactRefreshLogBox app turbo Import trace when module not found in layout 1`] = `
+"./app/module.js:1:0
+Module not found: Can't resolve 'non-existing-module'
+> 1 | import "non-existing-module"
+    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+https://nextjs.org/docs/messages/module-not-found"
+`;
+
+exports[`ReactRefreshLogBox app turbo Should not show __webpack_exports__ when exporting anonymous arrow function 1`] = `
+"index.js (1:13) @ __TURBOPACK__default__export__
+
+> 1 | export default () => {
+    |           ^
+  2 | if (typeof window !== 'undefined') {
+  3 |   throw new Error('test')
+  4 | }"
+`;
+
+exports[`ReactRefreshLogBox app turbo boundaries 1`] = `
+"FunctionDefault.js (0:24) @ FunctionDefault
+
+  1 | export default function FunctionDefault() { throw new Error('no'); }"
+`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 1`] = `"Error: end http://nextjs.org"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 2`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 3`] = `"Error: http://nextjs.org start"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 4`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 5`] = `"Error: middle http://nextjs.org end"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 6`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 8`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox app turbo logbox: anchors links in error messages 9`] = `"http://example.com/"`;
+
+exports[`ReactRefreshLogBox app turbo module init error not shown 1`] = `
+"index.js (3:6) @ <unknown>
+
+  1 | // top offset for snapshot
+  2 | import * as React from 'react';
+> 3 | throw new Error('no')
+    |    ^
+  4 | class ClassDefault extends React.Component {
+  5 | render() {
+  6 |   return <h1>Default Export</h1>;"
+`;
+
+exports[`ReactRefreshLogBox app turbo should strip whitespace correctly with newline 1`] = `
+"index.js (6:30) @ onClick
+
+  4 |
+  5 | <p>index page</p>
+> 6 |
+    | ^
+  7 | <a onClick={() => {
+  8 |   throw new Error('idk')
+  9 | }}>"
+`;

test/development/acceptance/ReactRefreshLogBox.test.ts

@@ -5,6 +5,8 @@ import { describeVariants as describe } from 'next-test-utils'
 import path from 'path'
 import { outdent } from 'outdent'
 
+const IS_TURBOPACK = Boolean(process.env.TURBOPACK)
+
 describe.each(['default', 'turbo'])('ReactRefreshLogBox %s', () => {
   const { next } = nextTestSetup({
     files: new FileRef(path.join(__dirname, 'fixtures', 'default-template')),
@@ -211,7 +213,18 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox %s', () => {
 
     const source = await session.getRedboxSource()
     expect(next.normalizeTestDirContent(source)).toMatchInlineSnapshot(
-      next.normalizeSnapshot(`
+      next.normalizeSnapshot(
+        IS_TURBOPACK
+          ? `
+      "./index.js:7:1
+      Parsing ecmascript source code failed
+        5 |     div
+        6 |   )
+      > 7 | }
+          |  ^
+      Unexpected eof"
+    `
+          : `
         "./index.js
         Error: 
           x Unexpected token. Did you mean \`{'}'}\` or \`}\`?
@@ -237,7 +250,8 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox %s', () => {
         Import trace for requested module:
         ./index.js
         ./pages/index.js"
-      `)
+      `
+      )
     )
 
     await cleanup()
@@ -335,13 +349,18 @@ describe.each(['default', 'turbo'])('ReactRefreshLogBox %s', () => {
     await session.patch('index.module.css', `.button {`)
     expect(await session.hasRedbox(true)).toBe(true)
     const source = await session.getRedboxSource()
-    expect(source).toMatch('./index.module.css:1:1')
-    expect(source).toMatch('Syntax error: ')
-    expect(source).toMatch('Unclosed block')
+    expect(source).toMatch(
+      IS_TURBOPACK ? './index.module.css:1:8' : './index.module.css:1:1'
+    )
+    if (!IS_TURBOPACK) {
+      expect(source).toMatch('Syntax error: ')
+      expect(source).toMatch('Unclosed block')
+    }
     expect(source).toMatch('> 1 | .button {')
-    expect(source).toMatch('    | ^')
+    expect(source).toMatch(IS_TURBOPACK ? '    |         ^' : '    | ^')
 
-    // Not local error
+    // Checks for selectors that can't be prefixed.
+    // Selector "button" is not pure (pure selectors must contain at least one local class or id)
     await session.patch('index.module.css', `button {}`)
     expect(await session.hasRedbox(true)).toBe(true)
     const source2 = await session.getRedboxSource()

test/development/acceptance/__snapshots__/ReactRefreshLogBox.test.ts.snap

@@ -69,3 +69,49 @@ exports[`ReactRefreshLogBox default should strip whitespace correctly with newli
   10 |   click me
   11 | </a>"
 `;
+
+exports[`ReactRefreshLogBox turbo boundaries 1`] = `null`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 1`] = `"Error: end http://nextjs.org"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 2`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 3`] = `"Error: http://nextjs.org start"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 4`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 5`] = `"Error: middle http://nextjs.org end"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 6`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 8`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 9`] = `"http://example.com/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 11`] = `"http://nextjs.org/"`;
+
+exports[`ReactRefreshLogBox turbo logbox: anchors links in error messages 12`] = `"http://example.com/"`;
+
+exports[`ReactRefreshLogBox turbo module init error not shown 1`] = `
+"index.js (3:6) @ <unknown>
+
+  1 | // top offset for snapshot
+  2 | import * as React from 'react';
+> 3 | throw new Error('no')
+    |    ^
+  4 | class ClassDefault extends React.Component {
+  5 | render() {
+  6 |   return <h1>Default Export</h1>;"
+`;
+
+exports[`ReactRefreshLogBox turbo should strip whitespace correctly with newline 1`] = `
+"index.js (8:18) @ onClick
+
+   6 |
+   7 | <a onClick={() => {
+>  8 |   throw new Error('idk')
+     |^
+   9 | }}>
+  10 |   click me
+  11 | </a>"
+`;

test/integration/edge-runtime-configurable-guards/test/index.test.js

@@ -9,6 +9,7 @@ import {
   launchApp,
   nextBuild,
   nextStart,
+  retry,
   waitFor,
 } from 'next-test-utils'
 import { remove } from 'fs-extra'
@@ -83,19 +84,22 @@ describe('Edge runtime configurable guards', () => {
       const res = await fetchViaHTTP(context.appPort, middlewareUrl)
       await waitFor(500)
       expect(res.status).toBe(200)
-      expect(context.logs.output).toContain(
-        `Dynamic Code Evaluation (e. g. 'eval', 'new Function') not allowed in Edge Runtime`
-      )
+      await retry(async () => {
+        expect(context.logs.output).toContain(
+          `Dynamic Code Evaluation (e. g. 'eval', 'new Function') not allowed in Edge Runtime`
+        )
+      })
     })
 
     it('warns in dev for unallowed code', async () => {
       context.app = await launchApp(context.appDir, context.appPort, appOption)
       const res = await fetchViaHTTP(context.appPort, routeUrl)
-      await waitFor(500)
       expect(res.status).toBe(200)
-      expect(context.logs.output).toContain(
-        `Dynamic Code Evaluation (e. g. 'eval', 'new Function') not allowed in Edge Runtime`
-      )
+      await retry(async () => {
+        expect(context.logs.output).toContain(
+          `Dynamic Code Evaluation (e. g. 'eval', 'new Function') not allowed in Edge Runtime`
+        )
+      })
     })
     ;(process.env.TURBOPACK ? describe.skip : describe)(
       'production mode',

test/lib/next-modes/base.ts

@@ -8,7 +8,7 @@ import { ChildProcess } from 'child_process'
 import { createNextInstall } from '../create-next-install'
 import { Span } from 'next/src/trace'
 import webdriver from '../next-webdriver'
-import { renderViaHTTP, fetchViaHTTP } from 'next-test-utils'
+import { renderViaHTTP, fetchViaHTTP, waitFor } from 'next-test-utils'
 import cheerio from 'cheerio'
 import { BrowserInterface } from '../browsers/base'
 import escapeStringRegexp from 'escape-string-regexp'
@@ -447,7 +447,7 @@ export class NextInstance {
     // to connect the WebSocket and start watching.
     if (process.env.TURBOPACK) {
       require('console').log('fs dev delay before', filename)
-      await new Promise((resolve) => setTimeout(resolve, 500))
+      await waitFor(500)
     }
   }
   private async handleDevWatchDelayAfterChange(filename: string) {

test/lib/next-webdriver.ts

@@ -1,4 +1,4 @@
-import { getFullUrl } from 'next-test-utils'
+import { getFullUrl, waitFor } from 'next-test-utils'
 import os from 'os'
 import { BrowserInterface } from './browsers/base'
 
@@ -186,5 +186,12 @@ export default async function webdriver(
 
     console.log(`\n> Hydration complete for ${fullUrl}\n`)
   }
+
+  // This is a temporary workaround for turbopack starting watching too late.
+  // So we delay file changes by 500ms to give it some time
+  // to connect the WebSocket and start watching.
+  if (process.env.TURBOPACK) {
+    await waitFor(1000)
+  }
   return browser
 }