Use Replay for smoke tests (#10664)

redwoodjs · May 30, 2024 · 14d9535 · 14d9535
1 parent d42ac10
commit 14d9535
Show file tree

Hide file tree

Showing 5 changed files with 289 additions and 31 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -203,6 +203,8 @@ jobs:
     env:
       REDWOOD_CI: 1
       REDWOOD_VERBOSE_TELEMETRY: 1
+      BROWSER_INSTALL_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright install --with-deps chromium' || 'npx replayio install' }}
+      PLAYWRIGHT_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright test --project chromium' || 'npx playwright test --project replay-chromium' }}
 
     steps:
       - uses: actions/checkout@v4
@@ -218,22 +220,24 @@ jobs:
           REDWOOD_DISABLE_TELEMETRY: 1
           YARN_ENABLE_IMMUTABLE_INSTALLS: false
 
-      - name: 🎭 Install playwright dependencies
-        run: npx playwright install --with-deps chromium
+      - name: 🎭 Install Playwright dependencies
+        run: ${{ env.BROWSER_INSTALL_CMD }}
 
       - name: 🧑‍💻 Run dev smoke tests
         working-directory: ./tasks/smoke-tests/dev
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: '${{ steps.set-up-test-project.outputs.test-project-path }}'
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: 🔐 Run auth smoke tests
         working-directory: ./tasks/smoke-tests/auth
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-test-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: Run `rw build --no-prerender`
         run: |
@@ -247,24 +251,27 @@ jobs:
 
       - name: 🖥️ Run serve smoke tests
         working-directory: tasks/smoke-tests/serve
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-test-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: 📄 Run prerender smoke tests
         working-directory: tasks/smoke-tests/prerender
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-test-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: 📕 Run Storybook smoke tests
         working-directory: tasks/smoke-tests/storybook
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-test-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
   smoke-tests-skip:
     needs: detect-changes
@@ -444,6 +451,8 @@ jobs:
     env:
       REDWOOD_CI: 1
       REDWOOD_VERBOSE_TELEMETRY: 1
+      BROWSER_INSTALL_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright install --with-deps chromium' || 'npx replayio install' }}
+      PLAYWRIGHT_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright test --project chromium' || 'npx playwright test --project replay-chromium' }}
 
     steps:
       - uses: actions/checkout@v4
@@ -457,15 +466,16 @@ jobs:
           REDWOOD_DISABLE_TELEMETRY: 1
           YARN_ENABLE_IMMUTABLE_INSTALLS: false
 
-      - name: 🎭 Install playwright dependencies
-        run: npx playwright install --with-deps chromium
+      - name: 🎭 Install Playwright dependencies
+        run: ${{ env.BROWSER_INSTALL_CMD }}
 
       - name: 🐘 Run RSC serve smoke tests
         working-directory: tasks/smoke-tests/rsc
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-rsc-project.outputs.rsc-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       # TODO (RSC): This workflow times out on Windows. It looks as if the dev
       # server starts up ok, but it doesn't seem like the browser is rendering
@@ -477,10 +487,11 @@ jobs:
       - name: 🐘 Run RSC dev smoke tests
         if: matrix.os == 'false__ubuntu-latest'
         working-directory: tasks/smoke-tests/rsc-dev
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-rsc-project.outputs.rsc-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: 🌲 Set up RSA smoke test
         id: set-up-rsa-project
@@ -491,10 +502,11 @@ jobs:
 
       - name: 🐘 Run RSA smoke tests
         working-directory: tasks/smoke-tests/rsa
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-rsa-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: 🌲 Set up RSC Kitchen Sink smoke test
         id: set-up-rsc-kitchen-sink-project
@@ -505,10 +517,11 @@ jobs:
 
       - name: 🐘 Run RSC Kitchen Sink smoke tests
         working-directory: tasks/smoke-tests/rsc-kitchen-sink
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-rsc-kitchen-sink-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
   rsc-smoke-tests-skip:
     needs: detect-changes
@@ -539,6 +552,9 @@ jobs:
     env:
       REDWOOD_CI: 1
       REDWOOD_VERBOSE_TELEMETRY: 1
+      # Temp workaround for Replay specific failing tests
+      BROWSER_INSTALL_CMD: 'npx playwright install --with-deps chromium && npx replayio install'
+      PLAYWRIGHT_CMD: npx playwright test --project chromium
 
     steps:
       - uses: actions/checkout@v4
@@ -560,15 +576,18 @@ jobs:
         env:
           REDWOOD_DISABLE_TELEMETRY: 1
 
-      - name: 🎭 Install playwright dependencies
-        run: npx playwright install --with-deps chromium
+      - name: 🎭 Install Playwright dependencies
+        run: ${{ env.BROWSER_INSTALL_CMD }}
 
       - name: Run SSR [DEV] smoke tests
         working-directory: ./tasks/smoke-tests/streaming-ssr-dev
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: '${{ steps.set-up-test-project.outputs.test-project-path }}'
           REDWOOD_DISABLE_TELEMETRY: 1
+          # Temp workaround for Replay specific failing tests
+          DISABLE_REPLAY_UPLOAD: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: Build for production
         working-directory: ${{ steps.set-up-test-project.outputs.test-project-path }}
@@ -578,10 +597,13 @@ jobs:
 
       - name: Run SSR [PROD] smoke tests
         working-directory: ./tasks/smoke-tests/streaming-ssr-prod
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: '${{ steps.set-up-test-project.outputs.test-project-path }}'
           REDWOOD_DISABLE_TELEMETRY: 1
+          # Temp workaround for Replay specific failing tests
+          DISABLE_REPLAY_UPLOAD: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
   ssr-smoke-tests-skip:
     needs: detect-changes
@@ -611,6 +633,8 @@ jobs:
     env:
       REDWOOD_CI: 1
       REDWOOD_VERBOSE_TELEMETRY: 1
+      BROWSER_INSTALL_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright install --with-deps chromium' || 'npx replayio install' }}
+      PLAYWRIGHT_CMD: ${{ matrix.os == 'windows-latest' && 'npx playwright test --project chromium' || 'npx playwright test --project replay-chromium' }}
 
     steps:
       - uses: actions/checkout@v4
@@ -632,15 +656,16 @@ jobs:
         env:
           REDWOOD_DISABLE_TELEMETRY: 1
 
-      - name: 🎭 Install playwright dependencies
-        run: npx playwright install --with-deps chromium
+      - name: 🎭 Install Playwright dependencies
+        run: ${{ env.BROWSER_INSTALL_CMD }}
 
       - name: Run Fragments dev smoke tests
         working-directory: ./tasks/smoke-tests/fragments-dev
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: '${{ steps.set-up-test-project.outputs.test-project-path }}'
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: Build for production
         working-directory: ${{ steps.set-up-test-project.outputs.test-project-path }}
@@ -650,10 +675,11 @@ jobs:
 
       - name: Run Fragments serve smoke tests
         working-directory: ./tasks/smoke-tests/fragments-serve
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: '${{ steps.set-up-test-project.outputs.test-project-path }}'
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
       - name: Enable trusted-documents in Fragments test project
         run: npx -y tsx ./tasks/test-project/set-up-trusted-documents ${{ steps.set-up-test-project.outputs.test-project-path }}
@@ -662,10 +688,11 @@ jobs:
 
       - name: 📄 Run prerender smoke tests against Fragments test project (with trusted-documents enabled)
         working-directory: tasks/smoke-tests/prerender
-        run: npx playwright test
+        run: ${{ env.PLAYWRIGHT_CMD }}
         env:
           REDWOOD_TEST_PROJECT_PATH: ${{ steps.set-up-test-project.outputs.test-project-path }}
           REDWOOD_DISABLE_TELEMETRY: 1
+          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
 
   fragments-smoke-tests-skip:
     needs: detect-changes

diff --git a/package.json b/package.json
@@ -67,6 +67,7 @@
     "@faker-js/faker": "8.4.1",
     "@npmcli/arborist": "7.5.2",
     "@playwright/test": "1.44.0",
+    "@replayio/playwright": "^3.0.1",
     "@testing-library/jest-dom": "6.4.5",
     "@testing-library/react": "14.3.1",
     "@testing-library/user-event": "14.5.2",

diff --git a/tasks/smoke-tests/basePlaywright.config.ts b/tasks/smoke-tests/basePlaywright.config.ts
@@ -2,6 +2,7 @@ import * as fs from 'node:fs'
 
 import type { PlaywrightTestConfig } from '@playwright/test'
 import { devices } from '@playwright/test'
+import { devices as replayDevices, replayReporter } from '@replayio/playwright'
 
 // See https://playwright.dev/docs/test-configuration#global-configuration
 export const basePlaywrightConfig: PlaywrightTestConfig = {
@@ -22,6 +23,10 @@ export const basePlaywrightConfig: PlaywrightTestConfig = {
       use: { ...devices['Desktop Chrome'] },
       dependencies: fs.existsSync('./tests/setup.ts') ? ['setup'] : undefined,
     },
+    {
+      name: 'replay-chromium',
+      use: { ...replayDevices['Replay Chromium'] },
+    },
 
     // {
     //   name: 'firefox',
@@ -34,5 +39,11 @@ export const basePlaywrightConfig: PlaywrightTestConfig = {
     // },
   ],
 
-  reporter: 'list',
+  reporter: [
+    replayReporter({
+      apiKey: process.env.REPLAY_API_KEY,
+      upload: !!process.env.CI || !!process.env.DISABLE_REPLAY_UPLOAD,
+    }),
+    ['line'],
+  ],
 }
diff --git a/tasks/smoke-tests/smoke-tests.mjs b/tasks/smoke-tests/smoke-tests.mjs
@@ -94,7 +94,7 @@ async function parseArgs() {
     playwrightOptions: {
       description: `Options to forward to ${chalk.cyan('npx playwright test')}`,
       type: /** @type {const} */ ('string'),
-      default: '',
+      default: '--project=chromium',
     },
 
     help: {