test: Refactor E2E test running (#8497)

This refactors how we run E2E tests, to a more simple approach.

.github/CANARY_FAILURE_TEMPLATE.md

@@ -1,5 +1,5 @@
 ---
-title: Canary tests failed
-labels: 'Status: Untriaged, Type: Tests'
+title: '{{ env.TITLE }}'
+labels: 'Type: Tests'
 ---
 Canary tests failed: {{ env.RUN_LINK }}

.github/workflows/build.yml

@@ -43,6 +43,7 @@ env:
     ${{ github.workspace }}/packages/utils/esm
 
   BUILD_CACHE_KEY: ${{ github.event.inputs.commit || github.sha }}
+  BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }}
 
   # GH will use the first restore-key it finds that matches
   # So it will start by looking for one from the same branch, else take the newest one it can find elsewhere
@@ -729,20 +730,82 @@ jobs:
           cd packages/remix
           yarn test:integration:ci
 
+  job_e2e_prepare:
+    name: Prepare E2E tests
+    if:
+      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
+      github.actor != 'dependabot[bot]'
+    needs: [job_get_metadata, job_build]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
+    steps:
+      - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'package.json'
+      - name: Restore caches
+        uses: ./.github/actions/restore-cache
+        env:
+          DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check tarball cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
+      - name: Build tarballs
+        run: yarn build:tarball
+
   job_e2e_tests:
-    name: E2E (Shard ${{ matrix.shard }}) Tests
+    name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
     # Dependabot PRs sadly also don't have access to secrets, so we skip them as well
     if:
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
-    needs: [job_get_metadata, job_build]
+    needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
-    timeout-minutes: 30
+    timeout-minutes: 10
+    env:
+      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
+      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # Needed because some apps expect a certain prefix
+      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
+      E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
+        test-application:
+          [
+            'node-express-app',
+            'create-react-app',
+            'create-next-app',
+            'create-remix-app',
+            'nextjs-app-dir',
+            'react-create-hash-router',
+            'standard-frontend-react',
+            'standard-frontend-react-tracing-import',
+            'sveltekit',
+          ]
+        build-command:
+          - false
+        label:
+          - false
+        # Add any variations of a test app here
+        # You should provide an alternate build-command as well as a matching label
+        include:
+          - test-application: 'create-react-app'
+            build-command: 'test:build-ts3.8'
+            label: 'create-react-app (TS 3.8)'
+          - test-application: 'standard-frontend-react'
+            build-command: 'test:build-ts3.8'
+            label: 'standard-frontend-react (TS 3.8)'
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -761,22 +824,36 @@ jobs:
         env:
           DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
 
+      - name: Restore tarball cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
+
       - name: Get node version
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
-      - name: Run E2E tests
+
+      - name: Validate Verdaccio
+        run: yarn test:validate
+        working-directory: packages/e2e-tests
+
+      - name: Prepare Verdaccio
+        run: yarn test:prepare
+        working-directory: packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
-          E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-          E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-          E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
-          E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
-          E2E_TEST_SHARD: ${{ matrix.shard }}
-          E2E_TEST_SHARD_AMOUNT: 4
-        run: |
-          cd packages/e2e-tests
-          yarn test:e2e
+
+      - name: Build E2E app
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn ${{ matrix.build-command || 'test:build' }}
+
+      - name: Run E2E test
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn test:assert
 
   job_required_tests:
     name: All required tests passed or skipped

.github/workflows/canary.yml

@@ -17,47 +17,123 @@ permissions:
   issues: write
 
 jobs:
-  job_canary_test:
-    name: Canary Tests
+  job_e2e_prepare:
+    name: Prepare E2E Canary tests
     runs-on: ubuntu-20.04
-    timeout-minutes: 60
+    timeout-minutes: 30
     steps:
-      - name: 'Check out current commit'
+      - name: Check out current commit
         uses: actions/checkout@v3
         with:
           ref: ${{ env.HEAD_COMMIT }}
-      - uses: pnpm/action-setup@v2
-        with:
-          version: 8.3.1
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
           node-version-file: 'package.json'
+      - name: Check canary cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/packages/*/*.tgz
+            ${{ github.workspace }}/node_modules
+            ${{ github.workspace }}/packages/*/node_modules
+          key: canary-${{ env.HEAD_COMMIT }}
       - name: Install dependencies
-        run: yarn install --ignore-engines --frozen-lockfile
+        run: yarn install
       - name: Build packages
         run: yarn build
+
+      - name: Build tarballs
+        run: yarn build:tarball
+
+  job_e2e_tests:
+    name: E2E ${{ matrix.label }} Test
+    needs: [job_e2e_prepare]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
+    env:
+      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
+      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # Needed because certain apps expect a certain prefix
+      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
+      E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - test-application: 'create-react-app'
+            build-command: 'test:build-canary'
+            label: 'create-react-app (canary)'
+          - test-application: 'nextjs-app-dir'
+            build-command: 'test:build-canary'
+            label: 'nextjs-app-dir (canary)'
+          - test-application: 'nextjs-app-dir'
+            build-command: 'test:build-latest'
+            label: 'nextjs-app-dir (latest)'
+          - test-application: 'react-create-hash-router'
+            build-command: 'test:build-canary'
+            label: 'react-create-hash-router (canary)'
+          - test-application: 'standard-frontend-react'
+            build-command: 'test:build-canary'
+            label: 'standard-frontend-react (canary)'
+
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8.3.1
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'package.json'
+
+      - name: Restore canary cache
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            ${{ github.workspace }}/packages/*/*.tgz
+            ${{ github.workspace }}/node_modules
+            ${{ github.workspace }}/packages/*/node_modules
+          key: canary-${{ env.HEAD_COMMIT }}
+
       - name: Get node version
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
-      - name: Run Canary Tests
+
+      - name: Validate Verdaccio
+        run: yarn test:validate
+        working-directory: packages/e2e-tests
+
+      - name: Prepare Verdaccio
+        run: yarn test:prepare
+        working-directory: packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
-          E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-          E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-          E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
-          E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
-          CANARY_E2E_TEST: 'yes'
-        run: |
-          cd packages/e2e-tests
-          yarn test:e2e
+
+      - name: Build E2E app
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn ${{ matrix.build-command }}
+
+      - name: Run E2E test
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn test:assert
+
       - name: Create Issue
         if: failure() && github.event_name == 'schedule'
         uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          TITLE: ${{ matrix.label }} Test Failed
         with:
           filename: .github/CANARY_FAILURE_TEMPLATE.md
           update_existing: true
@@ -98,7 +174,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUN_LINK: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          TITLE: Ember Canary ${{ matrix.scenario }} Test Failed
         with:
           filename: .github/CANARY_FAILURE_TEMPLATE.md
           update_existing: true
-          title: 'Ember Canary tests failed'

packages/e2e-tests/.gitignore

@@ -1,2 +1,4 @@
 .env
 tmp
+.tmp_build_output
+pnpm-lock.yaml

packages/e2e-tests/README.md

@@ -15,6 +15,12 @@ Prerequisites: Docker
 yarn test:e2e
 ```
 
+Or run only a single E2E test app:
+
+```bash
+yarn test:run <app-name>
+```
+
 ## How they work
 
 Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we
@@ -43,40 +49,14 @@ cat > test-applications/my-new-test-application/.npmrc << EOF
 @sentry:registry=http://localhost:4873
 @sentry-internal:registry=http://localhost:4873
 EOF
-
-# Add a test recipe file to the test application
-touch test-applications/my-new-test-application/test-recipe.json
-```
-
-To get you started with the recipe, you can copy the following into `test-recipe.json`:
-
-```json
-{
-  "$schema": "../../test-recipe-schema.json",
-  "testApplicationName": "My New Test Application",
-  "buildCommand": "pnpm install",
-  "tests": [
-    {
-      "testName": "My new test",
-      "testCommand": "pnpm test",
-      "timeoutSeconds": 60
-    }
-  ]
-}
 ```
 
-The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the
-fields:
+Make sure to add a `test:build` and `test:assert` command to the new app's `package.json` file.
 
-- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If
-  this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run. In
-  the example above, we use the `--pure-lockfile` flag to install dependencies without modifiying the lockfile so that
-  there aren't any changes in the git worktree after running the tests.
-- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a
-  non-zero exit code, it counts as a failed test.
-- A test timeout can be configured via `timeoutSeconds`, it defaults to `60`.
+Add the new test app to `test-application` matrix in `.github/workflows/build.yml` for the `job_e2e_tests` job. If you
+want to run a canary test, add it to the `canary.yml` workflow.
 
-**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at
+**An important thing to note:** In the context of the build/test commands the fake test registry is available at
 `http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current
 branch. This means we can install the packages from this registry via the `.npmrc` configuration as seen above. If you
 add Sentry dependencies to your test application, you should set the dependency versions set to `latest || *` in order
@@ -89,7 +69,9 @@ for it to work with both regular and prerelease versions:
   "version": "1.0.0",
   "private": true,
   "scripts": {
-    "test": "echo \"Hello world!\""
+    "test": "echo \"Hello world!\"",
+    "test:build": "pnpm install",
+    "test:assert": "pnpm test"
   },
   "dependencies": {
     "@sentry/node": "latest || *"