Add snapshot testing on e2e test failure (#24672)

We have a currently unreproducible flaky e2e test. This PR captures snapshots on e2e test failures so we can better debug flaky e2e tests that don't fail locally.
facebook · Jun 6, 2022 · 254b49e · 254b49e
1 parent 1cd90d2
commit 254b49e
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 4 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -291,6 +291,8 @@ jobs:
       - run:
           name: Cleanup build regression folder
           command: rm -r ./build-regression
+      - store_artifacts:
+          path: ./tmp/screenshots
 
   yarn_lint_build:
     docker: *docker

diff --git a/packages/react-devtools-inline/__tests__/__e2e__/components.test.js b/packages/react-devtools-inline/__tests__/__e2e__/components.test.js
@@ -210,7 +210,6 @@ test.describe('Components', () => {
     let count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 4');
 
-    await focusComponentSearch();
     page.keyboard.insertText('Item');
     count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 3');

diff --git a/packages/react-devtools-inline/playwright.config.js b/packages/react-devtools-inline/playwright.config.js
@@ -18,9 +18,10 @@ const config = {
     react_version: process.env.REACT_VERSION
       ? semver.coerce(process.env.REACT_VERSION).version
       : reactVersion,
+    trace: 'retain-on-failure',
   },
   // Some of our e2e tests can be flaky. Retry tests to make sure the error isn't transient
-  retries: 2,
+  retries: 3,
 };
 
 module.exports = config;
diff --git a/scripts/circleci/run_devtools_e2e_tests.js b/scripts/circleci/run_devtools_e2e_tests.js
@@ -9,6 +9,7 @@ const ROOT_PATH = join(__dirname, '..', '..');
 const reactVersion = process.argv[2];
 const inlinePackagePath = join(ROOT_PATH, 'packages', 'react-devtools-inline');
 const shellPackagePath = join(ROOT_PATH, 'packages', 'react-devtools-shell');
+const screenshotPath = join(ROOT_PATH, 'tmp', 'screenshots');
 
 let buildProcess = null;
 let serverProcess = null;
@@ -115,9 +116,11 @@ function runTestShell() {
 async function runEndToEndTests() {
   logBright('Running e2e tests');
   if (!reactVersion) {
-    testProcess = spawn('yarn', ['test:e2e'], {cwd: inlinePackagePath});
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
+      cwd: inlinePackagePath,
+    });
   } else {
-    testProcess = spawn('yarn', ['test:e2e'], {
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
       cwd: inlinePackagePath,
       env: {...process.env, REACT_VERSION: reactVersion},
     });