[test optimization] Add faulty logic to EFD in playwright (#6737)

Author: juan-fernandez

integration-tests/playwright/playwright.spec.js

@@ -736,6 +736,51 @@ versions.forEach((version) => {
           receiverPromise,
         ])
       })
+
+      it('does not run EFD if the percentage of new tests is too high', async () => {
+        receiver.setSettings({
+          early_flake_detection: {
+            enabled: true,
+            slow_test_retries: {
+              '5s': NUM_RETRIES_EFD
+            },
+            faulty_session_threshold: 0
+          },
+          known_tests_enabled: true
+        })
+
+        receiver.setKnownTests({ playwright: {} })
+
+        childProcess = exec(
+          './node_modules/.bin/playwright test -c playwright.config.js',
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              PW_BASE_URL: `http://localhost:${webAppPort}`
+            },
+            stdio: 'pipe'
+          }
+        )
+
+        await Promise.all([
+          once(childProcess, 'exit'),
+          receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const tests = events.filter(event => event.type === 'test').map(event => event.content)
+
+              const testSession = events.find(event => event.type === 'test_session_end').content
+              assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED)
+              assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty')
+
+              const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
+              assert.equal(newTests.length, 0)
+              const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
+              assert.equal(retriedTests.length, 0)
+            })
+        ])
+      })
     })
 
     it('does not crash when maxFailures=1 and there is an error', (done) => {

packages/datadog-instrumentations/src/playwright.js

@@ -7,7 +7,8 @@ const shimmer = require('../../datadog-shimmer')
 const {
   parseAnnotations,
   getTestSuitePath,
-  PLAYWRIGHT_WORKER_TRACE_PAYLOAD_CODE
+  PLAYWRIGHT_WORKER_TRACE_PAYLOAD_CODE,
+  getIsFaultyEarlyFlakeDetection
 } = require('../../dd-trace/src/plugins/util/test')
 const log = require('../../dd-trace/src/log')
 const { DD_MAJOR } = require('../../../version')
@@ -52,6 +53,7 @@ let isKnownTestsEnabled = false
 let isEarlyFlakeDetectionEnabled = false
 let earlyFlakeDetectionNumRetries = 0
 let isEarlyFlakeDetectionFaulty = false
+let earlyFlakeDetectionFaultyThreshold = 0
 let isFlakyTestRetriesEnabled = false
 let flakyTestRetriesCount = 0
 let knownTests = {}
@@ -542,6 +544,7 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) {
         isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
         isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
         earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
+        earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
         isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled
         flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount
         isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
@@ -846,40 +849,52 @@ addHook({
     if (isKnownTestsEnabled) {
       const newTests = allTests.filter(isNewTest)
 
-      /**
-       * We could repeat the logic of `applyRepeatEachIndex` here, but it'd be more risky
-       * as playwright could change it at any time.
-       *
-       * `applyRepeatEachIndex` goes through all the tests in a suite and applies the "repeat" logic
-       * for a single repeat index.
-       *
-       * This means that the clone logic is cumbersome:
-       * - we grab the unique file suites that have new tests
-       * - we store its project suite
-       * - we clone each of these file suites for each repeat index
-       * - we execute `applyRepeatEachIndex` for each of these cloned file suites
-       * - we add the cloned file suites to the project suite
-       */
-
-      const fileSuitesWithNewTestsToProjects = new Map()
-      newTests.forEach(newTest => {
-        newTest._ddIsNew = true
-        if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped' && !newTest._ddIsModified) {
-          const fileSuite = getSuiteType(newTest, 'file')
-          if (!fileSuitesWithNewTestsToProjects.has(fileSuite)) {
-            fileSuitesWithNewTestsToProjects.set(fileSuite, getSuiteType(newTest, 'project'))
+      const isFaulty = getIsFaultyEarlyFlakeDetection(
+        allTests.map(test => getTestSuitePath(test._requireFile, rootDir)),
+        knownTests.playwright,
+        earlyFlakeDetectionFaultyThreshold
+      )
+
+      if (isFaulty) {
+        isEarlyFlakeDetectionEnabled = false
+        isKnownTestsEnabled = false
+        isEarlyFlakeDetectionFaulty = true
+      } else {
+        /**
+         * We could repeat the logic of `applyRepeatEachIndex` here, but it'd be more risky
+         * as playwright could change it at any time.
+         *
+         * `applyRepeatEachIndex` goes through all the tests in a suite and applies the "repeat" logic
+         * for a single repeat index.
+         *
+         * This means that the clone logic is cumbersome:
+         * - we grab the unique file suites that have new tests
+         * - we store its project suite
+         * - we clone each of these file suites for each repeat index
+         * - we execute `applyRepeatEachIndex` for each of these cloned file suites
+         * - we add the cloned file suites to the project suite
+         */
+
+        const fileSuitesWithNewTestsToProjects = new Map()
+        newTests.forEach(newTest => {
+          newTest._ddIsNew = true
+          if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped' && !newTest._ddIsModified) {
+            const fileSuite = getSuiteType(newTest, 'file')
+            if (!fileSuitesWithNewTestsToProjects.has(fileSuite)) {
+              fileSuitesWithNewTestsToProjects.set(fileSuite, getSuiteType(newTest, 'project'))
+            }
           }
-        }
-      })
+        })
 
-      for (const [fileSuite, projectSuite] of fileSuitesWithNewTestsToProjects.entries()) {
-        for (let repeatEachIndex = 1; repeatEachIndex <= earlyFlakeDetectionNumRetries; repeatEachIndex++) {
-          const copyFileSuite = deepCloneSuite(fileSuite, isNewTest, [
-            '_ddIsNew',
-            '_ddIsEfdRetry'
-          ])
-          applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1)
-          projectSuite._addSuite(copyFileSuite)
+        for (const [fileSuite, projectSuite] of fileSuitesWithNewTestsToProjects.entries()) {
+          for (let repeatEachIndex = 1; repeatEachIndex <= earlyFlakeDetectionNumRetries; repeatEachIndex++) {
+            const copyFileSuite = deepCloneSuite(fileSuite, isNewTest, [
+              '_ddIsNew',
+              '_ddIsEfdRetry'
+            ])
+            applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1)
+            projectSuite._addSuite(copyFileSuite)
+          }
         }
       }
     }