fix e2e deployment test action

.github/workflows/build_reusable.yml

@@ -49,6 +49,16 @@ on:
         required: true
         description: 'name of the step, to be used for the upload artifact unique key '
         type: string
+      timeout_minutes:
+        description: 'Timeout in minutes'
+        required: false
+        type: number
+        default: 30
+      runs_on_labels:
+        description: 'List of runner labels'
+        required: false
+        type: string
+        default: '["self-hosted", "linux", "x64", "metal"]'
 
 env:
   NAPI_CLI_VERSION: 2.14.7
@@ -69,15 +79,13 @@ env:
   DD_ENV: 'ci'
   TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
   NEXT_TEST_JOB: 1
+  VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
+  VERCEL_TEST_TEAM: vtest314-next-e2e-tests
 
 jobs:
   build:
-    timeout-minutes: 30
-    runs-on:
-      - 'self-hosted'
-      - 'linux'
-      - 'x64'
-      - 'metal'
+    timeout-minutes: ${{ inputs.timeout_minutes }}
+    runs-on: ${{ fromJson(inputs.runs_on_labels) }}
 
     outputs:
       input_step_key: ${{ steps.var.outputs.input_step_key }}
@@ -90,7 +98,12 @@ jobs:
           script: |
             core.setOutput('input_step_key', '${{ inputs.stepName }}'.toLowerCase().replaceAll(/[/.]/g, '-').trim('-'));
 
-      - run: fnm use --install-if-missing ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }}
+          check-latest: true
+
       - run: node -v
       - run: corepack enable
       - run: pwd
@@ -175,7 +188,7 @@ jobs:
       - run: turbo run get-test-timings -- --build ${{ github.sha }}
 
       - run: /bin/bash -c "${{ inputs.afterBuild }}"
-        timeout-minutes: 30
+        timeout-minutes: ${{ inputs.timeout_minutes }}
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4

.github/workflows/test_e2e_deploy_release.yml

@@ -0,0 +1,95 @@
+name: Test E2E (Vercel Deploy), scheduled
+
+on:
+  # run on every release/prerelease
+  release:
+    types: [published]
+  # allow triggering manually as well
+  workflow_dispatch:
+
+env:
+  VERCEL_TEST_TEAM: vtest314-next-e2e-tests
+  VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
+  SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_DEPLOY_SLACK_WEBHOOK_URL }}
+  DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DD_ENV: 'ci'
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'vercel'
+    steps:
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+
+      - name: Setup pnpm
+        run: corepack enable
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 25
+
+      - name: Setup test project
+        run: |
+          pnpm install
+          pnpm run build
+          node scripts/run-e2e-test-project-reset.mjs
+
+  test-deploy:
+    name: test deploy
+    needs: setup
+    uses: ./.github/workflows/build_reusable.yml
+    secrets: inherit
+    strategy:
+      fail-fast: true
+      matrix:
+        group: [1/5, 2/5, 3/5, 4/5, 5/5, 6/6]
+    with:
+      afterBuild: NEXT_TEST_MODE=deploy NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
+      skipNativeBuild: 'yes'
+      stepName: 'test-deploy-${{ matrix.group }}'
+      timeout_minutes: 180
+      runs_on_labels: '["ubuntu-latest"]'
+
+  report-test-results-to-datadog:
+    needs: test-deploy
+    if: ${{ always() }}
+
+    runs-on: ubuntu-latest
+    name: report test results to datadog
+    steps:
+      - name: Download test report artifacts
+        id: download-test-reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: test-reports-*
+          path: test
+          merge-multiple: true
+
+      - name: Upload test report to datadog
+        run: |
+          if [ -d ./test/test-junit-report ]; then
+            DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:deploy --service nextjs ./test/test-junit-report
+          fi
+
+  report-failure:
+    name: report failure to slack
+    needs: test-deploy
+    if: needs.test-deploy.conclusion == 'failure'
+    runs-on: ubuntu-latest
+    steps:
+      - name: send webhook
+        uses: slackapi/slack-github-action@v1.25.0
+        with:
+          payload: |
+            {
+              "commit_title": ${{ toJSON(github.event.workflow_run.display_title) }},
+              "commit_url": "github.com/${{ github.repository }}/commit/${{ github.event.workflow_run.head_sha }}",
+              "workflow_run_url": "github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/attempts/${{ github.event.workflow_run.run_attempt }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}

run-tests.js

@@ -227,7 +227,12 @@ async function main() {
     }
   }
 
-  console.log('Running tests with concurrency:', options.concurrency)
+  console.log(
+    'Running tests with concurrency:',
+    options.concurrency,
+    'in test mode',
+    process.env.NEXT_TEST_MODE
+  )
 
   /** @type TestFile[] */
   let tests = argv._.filter((arg) =>
@@ -470,7 +475,8 @@ ${ENDGROUP}`)
         RECORD_REPLAY: shouldRecordTestWithReplay,
         // run tests in headless mode by default
         HEADLESS: 'true',
-        TRACE_PLAYWRIGHT: 'true',
+        TRACE_PLAYWRIGHT:
+          process.env.NEXT_TEST_MODE === 'deploy' ? undefined : 'true',
         NEXT_TELEMETRY_DISABLED: '1',
         // unset CI env so CI behavior is only explicitly
         // tested when enabled
@@ -691,32 +697,6 @@ ${ENDGROUP}`)
         }
       }
 
-      // Emit test output if test failed or if we're continuing tests on error
-      if ((!passed || shouldContinueTestsOnError) && isTestJob) {
-        try {
-          const testsOutput = await fsp.readFile(
-            `${test.file}${RESULTS_EXT}`,
-            'utf8'
-          )
-          const obj = JSON.parse(testsOutput)
-          obj.processEnv = {
-            NEXT_TEST_MODE: process.env.NEXT_TEST_MODE,
-            HEADLESS: process.env.HEADLESS,
-          }
-          await outputSema.acquire()
-          if (GROUP) console.log(`${GROUP}Result as JSON for tooling`)
-          console.log(
-            `--test output start--`,
-            JSON.stringify(obj),
-            `--test output end--`
-          )
-          if (ENDGROUP) console.log(ENDGROUP)
-          outputSema.release()
-        } catch (err) {
-          console.log(`Failed to load test output`, err)
-        }
-      }
-
       sema.release()
       if (dirSema) dirSema.release()
     })

test/deploy-tests-manifest.json

@@ -0,0 +1,80 @@
+{
+  "version": 2,
+  "suites": {},
+  "rules": {
+    "include": [
+      "test/e2e/**/*.test.{t,j}s{,x}",
+      "test/production/**/*.test.{t,j}s{,x}"
+    ],
+    "exclude": [
+      "test/e2e/app-dir/app-client-cache/client-cache.original.test.ts",
+      "test/e2e/app-dir/app-routes/app-custom-routes.test.ts",
+      "test/e2e/app-dir/next-after-app/index.test.ts",
+      "test/e2e/app-dir/scss/nm-module-nested/nm-module-nested.test.ts",
+      "test/e2e/cancel-request/stream-cancel.test.ts",
+      "test/e2e/edge-pages-support/edge-document.test.ts",
+      "test/e2e/new-link-behavior/material-ui.test.ts",
+      "test/e2e/react-dnd-compile/react-dnd-compile.test.ts",
+      "test/e2e/next-test/next-test.test.ts",
+      "test/e2e/skip-trailing-slash-redirect/index.test.ts",
+      "test/e2e/tsconfig-module-preserve/index.test.ts",
+      "test/e2e/app-dir/app-compilation/index.test.ts",
+      "test/e2e/app-dir/parallel-route-not-found-params/parallel-route-not-found-params.test.ts",
+      "test/e2e/app-dir/ppr-navigations/loading-tsx-no-partial-rendering/loading-tsx-no-partial-rendering.test.ts",
+      "test/e2e/app-dir/ppr/ppr.test.ts",
+      "test/e2e/app-dir/rsc-webpack-loader/rsc-webpack-loader.test.ts",
+      "test/e2e/app-dir/scss/compilation-and-prefixing/compilation-and-prefixing.test.ts",
+      "test/e2e/app-dir/typeof-window/typeof-window.test.ts",
+      "test/e2e/app-dir/webpack-loader-conditions/webpack-loader-conditions.test.ts",
+      "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts",
+      "test/e2e/edge-compiler-module-exports-preference/index.test.ts",
+      "test/e2e/swc-warnings/index.test.ts",
+      "test/e2e/third-parties/index.test.ts",
+      "test/e2e/useselectedlayoutsegment-s-in-pages-router/useselectedlayoutsegment-s-in-pages-router.test.ts",
+      "test/e2e/app-dir/app-prefetch-false-loading/app-prefetch-false-loading.test.ts",
+      "test/e2e/app-dir/app-client-cache/client-cache.experimental.test.ts",
+      "test/e2e/app-dir/app-routes-client-component/app-routes-client-component.test.ts",
+      "test/e2e/app-dir/app-routes/app-custom-route-base-path.test.ts",
+      "test/e2e/app-dir/mdx/mdx.test.ts",
+      "test/e2e/app-dir/modularizeimports/modularizeimports.test.ts",
+      "test/e2e/app-dir/navigation/navigation.test.ts",
+      "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts",
+      "test/e2e/app-dir/ppr-navigations/avoid-popstate-flash/avoid-popstate-flash.test.ts",
+      "test/e2e/app-dir/server-components-externals/index.test.ts",
+      "test/e2e/app-dir/third-parties/basic.test.ts",
+      "test/e2e/edge-can-read-request-body/index.test.ts",
+      "test/e2e/esm-externals/esm-externals.test.ts",
+      "test/e2e/i18n-data-route/i18n-data-route.test.ts",
+      "test/e2e/next-phase/index.test.ts",
+      "test/e2e/app-dir/actions-navigation/index.test.ts",
+      "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts",
+      "test/e2e/app-dir/conflicting-page-segments/conflicting-page-segments.test.ts",
+      "test/e2e/app-dir/interception-route-prefetch-cache/interception-route-prefetch-cache.test.ts",
+      "test/e2e/app-dir/missing-suspense-with-csr-bailout/missing-suspense-with-csr-bailout.test.ts",
+      "test/e2e/app-dir/options-request/options-request.test.ts",
+      "test/e2e/app-dir/next-image/next-image-proxy.test.ts",
+      "test/e2e/app-dir/ppr-navigations/stale-prefetch-entry/stale-prefetch-entry.test.ts",
+      "test/e2e/app-dir/parallel-routes-revalidation/parallel-routes-revalidation.test.ts",
+      "test/e2e/app-dir/revalidate-dynamic/revalidate-dynamic.test.ts",
+      "test/e2e/app-dir/scss/npm-import-nested/npm-import-nested.test.ts",
+      "test/e2e/app-dir/syntax-highlighter-crash/syntax-highlighter-crash.test.ts",
+      "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts",
+      "test/e2e/new-link-behavior/stitches.test.ts",
+      "test/e2e/next-image-forward-ref/index.test.ts",
+      "test/e2e/react-compiler/react-compiler.test.ts",
+      "test/e2e/revalidate-reason/revalidate-reason.test.ts",
+      "test/e2e/app-dir/app-static/app-static.test.ts",
+      "test/e2e/app-dir/actions/app-action.test.ts",
+      "test/e2e/app-dir/i18n-hybrid/i18n-hybrid.test.js",
+      "test/e2e/app-dir/metadata/metadata.test.ts",
+      "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts",
+      "test/e2e/app-dir/scss/nm-module/nm-module.test.ts",
+      "test/e2e/app-dir/static-shell-debugging/static-shell-debugging.test.ts",
+      "test/e2e/basepath.test.ts",
+      "test/e2e/postcss-config-cjs/index.test.ts",
+      "test/e2e/socket-io/index.test.js",
+      "test/e2e/middleware-matcher/index.test.ts",
+      "test/e2e/next-script/index.test.ts"
+    ]
+  }
+}

test/get-test-filter.js

@@ -7,6 +7,11 @@ function getTestFilter() {
     : null
   if (!manifest) return null
 
+  console.log(
+    'Filtering tests using manifest:',
+    process.env.NEXT_EXTERNAL_TESTS_FILTERS
+  )
+
   // For the legacy manifest without a version, we assume it's a complete list
   // of all the tests.
   if (!manifest.version || typeof manifest.version !== 'number') {