Skip to content

Commit a1725a1

Browse files
mydeamydea
authored
ci: Add optional E2E tests for sending to Sentry (#12259)
This adds a new E2E test `react-send-to-sentry` that is run optionally. For now this is the same as the old `standard-frontend-react` test - eventually we can/should update that test (and others) to stop sending to sentry. This test will run in a separate group that we do not block merging on when it fails. For now, there is one browser and one node test that checks that they send events successfully to Sentry - IMHO that should cover stuff decently for now. I also made the source maps debug ID test optional, as that inherently sends to Sentry. We can in follow ups get rid of all the event sending stuff from the remaining E2E tests. Part of #11910
1 parent dc6bbec commit a1725a1

File tree

21 files changed

+1092
-1
lines changed

21 files changed

+1092
-1
lines changed

‎.github/workflows/build.yml

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1001,7 +1001,6 @@ jobs:
10011001
'create-remix-app-v2',
10021002
'create-remix-app-express',
10031003
'create-remix-app-express-vite-dev',
1004-
'debug-id-sourcemaps',
10051004
'node-express-esm-loader',
10061005
'node-express-esm-preload',
10071006
'node-express-esm-without-loader',
@@ -1113,6 +1112,90 @@ jobs:
11131112
directory: dist
11141113
workingDirectory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
11151114

1115+
job_optional_e2e_tests:
1116+
name: E2E ${{ matrix.label || matrix.test-application }} Test
1117+
# We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
1118+
# We need to add the `always()` check here because the previous step has this as well :(
1119+
# See: https://github.com/actions/runner/issues/2205
1120+
if:
1121+
always() && needs.job_e2e_prepare.result == 'success' &&
1122+
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
1123+
needs: [job_get_metadata, job_build, job_e2e_prepare]
1124+
runs-on: ubuntu-20.04
1125+
timeout-minutes: 10
1126+
env:
1127+
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
1128+
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
1129+
# Needed because some apps expect a certain prefix
1130+
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
1131+
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
1132+
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
1133+
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
1134+
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
1135+
strategy:
1136+
fail-fast: false
1137+
matrix:
1138+
is_dependabot:
1139+
- ${{ github.actor == 'dependabot[bot]' }}
1140+
test-application:
1141+
[
1142+
'react-send-to-sentry',
1143+
'node-express-send-to-sentry',
1144+
'debug-id-sourcemaps',
1145+
]
1146+
build-command:
1147+
- false
1148+
label:
1149+
- false
1150+
1151+
steps:
1152+
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
1153+
uses: actions/checkout@v4
1154+
with:
1155+
ref: ${{ env.HEAD_COMMIT }}
1156+
- uses: pnpm/action-setup@v2
1157+
with:
1158+
version: 8.3.1
1159+
- name: Set up Node
1160+
uses: actions/setup-node@v4
1161+
with:
1162+
node-version-file: 'dev-packages/e2e-tests/package.json'
1163+
- name: Restore caches
1164+
uses: ./.github/actions/restore-cache
1165+
env:
1166+
DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
1167+
1168+
- name: Restore tarball cache
1169+
uses: actions/cache/restore@v4
1170+
with:
1171+
path: ${{ github.workspace }}/packages/*/*.tgz
1172+
key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
1173+
1174+
- name: Get node version
1175+
id: versions
1176+
run: |
1177+
echo "echo node=$(jq -r '.volta.node' dev-packages/e2e-tests/package.json)" >> $GITHUB_OUTPUT
1178+
1179+
- name: Validate Verdaccio
1180+
run: yarn test:validate
1181+
working-directory: dev-packages/e2e-tests
1182+
1183+
- name: Prepare Verdaccio
1184+
run: yarn test:prepare
1185+
working-directory: dev-packages/e2e-tests
1186+
env:
1187+
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
1188+
1189+
- name: Build E2E app
1190+
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1191+
timeout-minutes: 5
1192+
run: yarn ${{ matrix.build-command || 'test:build' }}
1193+
1194+
- name: Run E2E test
1195+
working-directory: dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}
1196+
timeout-minutes: 5
1197+
run: yarn test:assert
1198+
11161199
job_profiling_e2e_tests:
11171200
name: E2E ${{ matrix.label || matrix.test-application }} Test
11181201
# We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dist

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/.npmrc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@sentry:registry=http://127.0.0.1:4873
2+
@sentry-internal:registry=http://127.0.0.1:4873

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/package.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"name": "node-express-send-to-sentry-app",
3+
"version": "1.0.0",
4+
"private": true,
5+
"scripts": {
6+
"build": "tsc",
7+
"start": "node dist/app.js",
8+
"test": "playwright test",
9+
"clean": "npx rimraf node_modules pnpm-lock.yaml",
10+
"test:build": "pnpm install && pnpm build",
11+
"test:assert": "pnpm test"
12+
},
13+
"dependencies": {
14+
"@sentry/core": "latest || *",
15+
"@sentry/node": "latest || *",
16+
"@sentry/types": "latest || *",
17+
"@types/express": "4.17.17",
18+
"@types/node": "18.15.1",
19+
"express": "4.19.2",
20+
"typescript": "4.9.5"
21+
},
22+
"devDependencies": {
23+
"@playwright/test": "^1.27.1"
24+
},
25+
"volta": {
26+
"extends": "../../package.json"
27+
}
28+
}

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/playwright.config.mjs

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { devices } from '@playwright/test';
2+
3+
// Fix urls not resolving to localhost on Node v17+
4+
// See: https://github.com/axios/axios/issues/3821#issuecomment-1413727575
5+
import { setDefaultResultOrder } from 'dns';
6+
setDefaultResultOrder('ipv4first');
7+
8+
const expressPort = 3030;
9+
10+
/**
11+
* See https://playwright.dev/docs/test-configuration.
12+
*/
13+
const config = {
14+
testDir: './tests',
15+
/* Maximum time one test can run for. */
16+
timeout: 150_000,
17+
expect: {
18+
/**
19+
* Maximum time expect() should wait for the condition to be met.
20+
* For example in `await expect(locator).toHaveText();`
21+
*/
22+
timeout: 5000,
23+
},
24+
/* Run tests in files in parallel */
25+
fullyParallel: true,
26+
/* Fail the build on CI if you accidentally left test.only in the source code. */
27+
forbidOnly: !!process.env.CI,
28+
/* Retry on CI only */
29+
retries: 0,
30+
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
31+
reporter: 'list',
32+
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
33+
use: {
34+
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
35+
actionTimeout: 0,
36+
37+
/* Base URL to use in actions like `await page.goto('/')`. */
38+
baseURL: `http://localhost:${expressPort}`,
39+
40+
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
41+
trace: 'on-first-retry',
42+
},
43+
44+
/* Configure projects for major browsers */
45+
projects: [
46+
{
47+
name: 'chromium',
48+
use: {
49+
...devices['Desktop Chrome'],
50+
},
51+
},
52+
// For now we only test Chrome!
53+
// {
54+
// name: 'firefox',
55+
// use: {
56+
// ...devices['Desktop Firefox'],
57+
// },
58+
// },
59+
// {
60+
// name: 'webkit',
61+
// use: {
62+
// ...devices['Desktop Safari'],
63+
// },
64+
// },
65+
],
66+
67+
/* Run your local dev server before starting the tests */
68+
webServer: [
69+
{
70+
command: 'pnpm start',
71+
port: expressPort,
72+
},
73+
],
74+
};
75+
76+
export default config;

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/src/app.ts

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import * as Sentry from '@sentry/node';
2+
3+
let lastTransactionId: string | undefined;
4+
5+
Sentry.init({
6+
environment: 'qa', // dynamic sampling bias to keep transactions
7+
dsn: process.env.E2E_TEST_DSN,
8+
includeLocalVariables: true,
9+
tracesSampleRate: 1,
10+
beforeSendTransaction(event) {
11+
lastTransactionId = event.event_id;
12+
return event;
13+
},
14+
});
15+
16+
import express from 'express';
17+
18+
const app = express();
19+
const port = 3030;
20+
21+
app.get('/test-success', function (req, res) {
22+
res.send({ version: 'v1' });
23+
});
24+
25+
app.get('/test-param/:param', function (req, res) {
26+
res.send({ paramWas: req.params.param });
27+
});
28+
29+
app.get('/test-transaction', function (req, res) {
30+
Sentry.withActiveSpan(null, async () => {
31+
Sentry.startSpan({ name: 'test-transaction', op: 'e2e-test' }, () => {
32+
Sentry.startSpan({ name: 'test-span' }, () => undefined);
33+
});
34+
35+
await Sentry.flush();
36+
37+
res.send({
38+
transactionId: lastTransactionId,
39+
});
40+
});
41+
});
42+
43+
app.get('/test-error', async function (req, res) {
44+
const exceptionId = Sentry.captureException(new Error('This is an error'));
45+
46+
await Sentry.flush(2000);
47+
48+
res.send({ exceptionId });
49+
});
50+
51+
app.get('/test-exception/:id', function (req, _res) {
52+
throw new Error(`This is an exception with id ${req.params.id}`);
53+
});
54+
55+
app.get('/test-local-variables-uncaught', function (req, res) {
56+
const randomVariableToRecord = Math.random();
57+
throw new Error(`Uncaught Local Variable Error - ${JSON.stringify({ randomVariableToRecord })}`);
58+
});
59+
60+
app.get('/test-local-variables-caught', function (req, res) {
61+
const randomVariableToRecord = Math.random();
62+
63+
let exceptionId: string;
64+
try {
65+
throw new Error('Local Variable Error');
66+
} catch (e) {
67+
exceptionId = Sentry.captureException(e);
68+
}
69+
70+
res.send({ exceptionId, randomVariableToRecord });
71+
});
72+
73+
Sentry.setupExpressErrorHandler(app);
74+
75+
// @ts-ignore
76+
app.use(function onError(err, req, res, next) {
77+
// The error id is attached to `res.sentry` to be returned
78+
// and optionally displayed to the user for support.
79+
res.statusCode = 500;
80+
res.end(res.sentry + '\n');
81+
});
82+
83+
app.listen(port, () => {
84+
console.log(`Example app listening on port ${port}`);
85+
});

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/tests/send-to-sentry.test.ts

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import { expect, test } from '@playwright/test';
2+
3+
const EVENT_POLLING_TIMEOUT = 90_000;
4+
5+
const authToken = process.env.E2E_TEST_AUTH_TOKEN;
6+
const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG;
7+
const sentryTestProject = process.env.E2E_TEST_SENTRY_TEST_PROJECT;
8+
9+
test('Sends exception to Sentry', async ({ baseURL }) => {
10+
const response = await fetch(`${baseURL}/test-error`);
11+
const { exceptionId } = await response.json();
12+
13+
const url = `https://sentry.io/api/0/projects/${sentryTestOrgSlug}/${sentryTestProject}/events/${exceptionId}/`;
14+
15+
console.log(`Polling for error eventId: ${exceptionId}`);
16+
17+
await expect
18+
.poll(
19+
async () => {
20+
const response = await fetch(url, { headers: { Authorization: `Bearer ${authToken}` } });
21+
return response.status;
22+
},
23+
{ timeout: EVENT_POLLING_TIMEOUT },
24+
)
25+
.toBe(200);
26+
});
27+
28+
test('Sends transaction to Sentry', async ({ baseURL }) => {
29+
const response = await fetch(`${baseURL}/test-transaction`);
30+
const { transactionId } = await response.json();
31+
32+
const url = `https://sentry.io/api/0/projects/${sentryTestOrgSlug}/${sentryTestProject}/events/${transactionId}/`;
33+
34+
console.log(`Polling for transaction eventId: ${transactionId}`);
35+
36+
await expect
37+
.poll(
38+
async () => {
39+
const response = await fetch(url, { headers: { Authorization: `Bearer ${authToken}` } });
40+
return response.status;
41+
},
42+
{ timeout: EVENT_POLLING_TIMEOUT },
43+
)
44+
.toBe(200);
45+
});

‎dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/tsconfig.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"compilerOptions": {
3+
"types": ["node"],
4+
"esModuleInterop": true,
5+
"lib": ["es2018"],
6+
"strict": true,
7+
"outDir": "dist"
8+
},
9+
"include": ["src/**/*.ts"]
10+
}

‎dev-packages/e2e-tests/test-applications/react-send-to-sentry/.gitignore

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2+
3+
# dependencies
4+
/node_modules
5+
/.pnp
6+
.pnp.js
7+
8+
# testing
9+
/coverage
10+
11+
# production
12+
/build
13+
14+
# misc
15+
.DS_Store
16+
.env.local
17+
.env.development.local
18+
.env.test.local
19+
.env.production.local
20+
21+
npm-debug.log*
22+
yarn-debug.log*
23+
yarn-error.log*
24+
25+
/test-results/
26+
/playwright-report/
27+
/playwright/.cache/
28+
29+
!*.d.ts

‎dev-packages/e2e-tests/test-applications/react-send-to-sentry/.npmrc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@sentry:registry=http://127.0.0.1:4873
2+
@sentry-internal:registry=http://127.0.0.1:4873

0 commit comments

Comments
 (0)