Skip to content

Commit 02c02a4

Browse files
ztannerztanner
authored
split out flake detection from utility to detect changed tests (vercel#67611)
This splits out the existing logic for detecting changed/added tests into a separate util, so it can be leveraged by vercel#67612. No changes in functionality, aside from replacing informational logs with `console.log` rather than `console.error` and a tweak to arg parsing.
1 parent 1309dee commit 02c02a4

File tree

3 files changed

+132
-100
lines changed

3 files changed

+132
-100
lines changed

‎.github/workflows/build_and_test.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ jobs:
291291

292292
uses: ./.github/workflows/build_reusable.yml
293293
with:
294-
afterBuild: node scripts/test-new-tests.mjs --dev-mode --group ${{ matrix.group }}
294+
afterBuild: node scripts/test-new-tests.mjs --flake-detection --mode dev --group ${{ matrix.group }}
295295
stepName: 'test-new-tests-dev-${{matrix.group}}'
296296

297297
secrets: inherit
@@ -308,7 +308,7 @@ jobs:
308308

309309
uses: ./.github/workflows/build_reusable.yml
310310
with:
311-
afterBuild: node scripts/test-new-tests.mjs --prod-mode --group ${{ matrix.group }}
311+
afterBuild: node scripts/test-new-tests.mjs --flake-detection --mode start --group ${{ matrix.group }}
312312
stepName: 'test-new-tests-start-${{matrix.group}}'
313313

314314
secrets: inherit

‎scripts/get-changed-tests.mjs

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
// @ts-check
2+
import fs from 'fs/promises'
3+
import execa from 'execa'
4+
import path from 'path'
5+
6+
/**
7+
* Detects changed tests files by comparing the current branch with `origin/canary`
8+
* Returns tests separated by test mode (dev/prod)
9+
*/
10+
export default async function getChangedTests() {
11+
let eventData = {}
12+
13+
/** @type import('execa').Options */
14+
const EXECA_OPTS = { shell: true }
15+
/** @type import('execa').Options */
16+
const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' }
17+
18+
try {
19+
eventData =
20+
JSON.parse(
21+
await fs.readFile(process.env.GITHUB_EVENT_PATH || '', 'utf8')
22+
)['pull_request'] || {}
23+
} catch (_) {}
24+
25+
const branchName =
26+
eventData?.head?.ref ||
27+
process.env.GITHUB_REF_NAME ||
28+
(await execa('git rev-parse --abbrev-ref HEAD', EXECA_OPTS)).stdout
29+
30+
const remoteUrl =
31+
eventData?.head?.repo?.full_name ||
32+
process.env.GITHUB_REPOSITORY ||
33+
(await execa('git remote get-url origin', EXECA_OPTS)).stdout
34+
35+
const isCanary =
36+
branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js')
37+
38+
if (isCanary) {
39+
console.log(`Skipping flake detection for canary`)
40+
return { devTests: [], prodTests: [] }
41+
}
42+
43+
try {
44+
await execa('git remote set-branches --add origin canary', EXECA_OPTS_STDIO)
45+
await execa('git fetch origin canary --depth=20', EXECA_OPTS_STDIO)
46+
} catch (err) {
47+
console.error(await execa('git remote -v', EXECA_OPTS_STDIO))
48+
console.error(`Failed to fetch origin/canary`, err)
49+
}
50+
51+
const changesResult = await execa(
52+
`git diff origin/canary --name-only`,
53+
EXECA_OPTS
54+
).catch((err) => {
55+
console.error(err)
56+
return { stdout: '', stderr: '' }
57+
})
58+
console.log(
59+
{
60+
branchName,
61+
remoteUrl,
62+
isCanary,
63+
},
64+
`\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}`
65+
)
66+
const changedFiles = changesResult.stdout.split('\n')
67+
68+
// run each test 3 times in each test mode (if E2E) with no-retrying
69+
// and if any fail it's flakey
70+
const devTests = []
71+
const prodTests = []
72+
73+
for (let file of changedFiles) {
74+
// normalize slashes
75+
file = file.replace(/\\/g, '/')
76+
const fileExists = await fs
77+
.access(path.join(process.cwd(), file), fs.constants.F_OK)
78+
.then(() => true)
79+
.catch(() => false)
80+
81+
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
82+
if (file.startsWith('test/e2e/')) {
83+
devTests.push(file)
84+
prodTests.push(file)
85+
} else if (file.startsWith('test/prod')) {
86+
prodTests.push(file)
87+
} else if (file.startsWith('test/development')) {
88+
devTests.push(file)
89+
}
90+
}
91+
}
92+
93+
console.log(
94+
'Detected tests:',
95+
JSON.stringify(
96+
{
97+
devTests,
98+
prodTests,
99+
},
100+
null,
101+
2
102+
)
103+
)
104+
105+
return { devTests, prodTests }
106+
}

‎scripts/test-new-tests.mjs

Lines changed: 24 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,30 @@
11
// @ts-check
2-
import fs from 'fs/promises'
32
import execa from 'execa'
4-
import path from 'path'
53
import yargs from 'yargs'
6-
4+
import getChangedTests from './get-changed-tests.mjs'
5+
6+
/**
7+
* Run tests for added/changed tests in the current branch
8+
* CLI Options:
9+
* --mode: test mode (dev, deploy, start)
10+
* --group: current group number / total groups
11+
* --flake-detection: run tests multiple times to detect flaky
12+
*/
713
async function main() {
814
let argv = await yargs(process.argv.slice(2))
9-
.boolean('dev-mode')
10-
.string('group').argv
15+
.string('mode')
16+
.string('group')
17+
.boolean('flake-detection').argv
18+
19+
let testMode = argv.mode
20+
const attempts = argv['flake-detection'] ? 3 : 1
21+
22+
if (testMode && !['dev', 'deploy', 'start'].includes(testMode)) {
23+
throw new Error(
24+
`Invalid test mode: ${testMode}. Must be one of: dev, deploy, start`
25+
)
26+
}
1127

12-
let testMode = argv.devMode ? 'dev' : 'start'
1328
const rawGroup = argv['group']
1429
let currentGroup = 1
1530
let groupTotal = 1
@@ -20,101 +35,12 @@ async function main() {
2035
.map((item) => Number(item))
2136
}
2237

23-
let eventData = {}
24-
2538
/** @type import('execa').Options */
2639
const EXECA_OPTS = { shell: true }
2740
/** @type import('execa').Options */
2841
const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' }
2942

30-
try {
31-
eventData =
32-
JSON.parse(
33-
await fs.readFile(process.env.GITHUB_EVENT_PATH || '', 'utf8')
34-
)['pull_request'] || {}
35-
} catch (_) {}
36-
37-
// detect changed test files
38-
const branchName =
39-
eventData?.head?.ref ||
40-
process.env.GITHUB_REF_NAME ||
41-
(await execa('git rev-parse --abbrev-ref HEAD', EXECA_OPTS)).stdout
42-
43-
const remoteUrl =
44-
eventData?.head?.repo?.full_name ||
45-
process.env.GITHUB_REPOSITORY ||
46-
(await execa('git remote get-url origin', EXECA_OPTS)).stdout
47-
48-
const isCanary =
49-
branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js')
50-
51-
if (isCanary) {
52-
console.error(`Skipping flake detection for canary`)
53-
return
54-
}
55-
56-
try {
57-
await execa('git remote set-branches --add origin canary', EXECA_OPTS_STDIO)
58-
await execa('git fetch origin canary --depth=20', EXECA_OPTS_STDIO)
59-
} catch (err) {
60-
console.error(await execa('git remote -v', EXECA_OPTS_STDIO))
61-
console.error(`Failed to fetch origin/canary`, err)
62-
}
63-
64-
const changesResult = await execa(
65-
`git diff origin/canary --name-only`,
66-
EXECA_OPTS
67-
).catch((err) => {
68-
console.error(err)
69-
return { stdout: '', stderr: '' }
70-
})
71-
console.error(
72-
{
73-
branchName,
74-
remoteUrl,
75-
isCanary,
76-
testMode,
77-
},
78-
`\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}`
79-
)
80-
const changedFiles = changesResult.stdout.split('\n')
81-
82-
// run each test 3 times in each test mode (if E2E) with no-retrying
83-
// and if any fail it's flakey
84-
const devTests = []
85-
const prodTests = []
86-
87-
for (let file of changedFiles) {
88-
// normalize slashes
89-
file = file.replace(/\\/g, '/')
90-
const fileExists = await fs
91-
.access(path.join(process.cwd(), file), fs.constants.F_OK)
92-
.then(() => true)
93-
.catch(() => false)
94-
95-
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
96-
if (file.startsWith('test/e2e/')) {
97-
devTests.push(file)
98-
prodTests.push(file)
99-
} else if (file.startsWith('test/prod')) {
100-
prodTests.push(file)
101-
} else if (file.startsWith('test/development')) {
102-
devTests.push(file)
103-
}
104-
}
105-
}
106-
107-
console.log(
108-
'Detected tests:',
109-
JSON.stringify(
110-
{
111-
devTests,
112-
prodTests,
113-
},
114-
null,
115-
2
116-
)
117-
)
43+
const { devTests, prodTests } = await getChangedTests()
11844

11945
let currentTests = testMode === 'dev' ? devTests : prodTests
12046

@@ -153,8 +79,8 @@ async function main() {
15379

15480
const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0']
15581

156-
for (let i = 0; i < 3; i++) {
157-
console.log(`\n\nRun ${i + 1} for ${testMode} tests`)
82+
for (let i = 0; i < attempts; i++) {
83+
console.log(`\n\nRun ${i + 1}/${attempts} for ${testMode} tests`)
15884
await execa('node', [...RUN_TESTS_ARGS, ...currentTests], {
15985
...EXECA_OPTS_STDIO,
16086
env: {

0 commit comments

Comments
 (0)