[test-optimization] [SDTEST-2024] Fix Playwright exit code status when it fails before running the test (#5733)

Author: Mariovido

integration-tests/ci-visibility/playwright-tests-exit-code/exit-code-test.js

@@ -0,0 +1,15 @@
+const { test, expect } = require('@playwright/test')
+// eslint-disable-next-line no-unused-vars
+const dummy = require('dummy') // This should not exist, so should throw an error
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('exit code', () => {
+  test('should exit with code 1', async ({ page }) => {
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello World'
+    ])
+  })
+})

integration-tests/playwright/playwright.spec.js

@@ -1488,6 +1488,37 @@ versions.forEach((version) => {
           runTest(done, { isRedirecting: true })
         })
       })
+
+      context('run session status', () => {
+        it('session status is not changed if it fails before running any test', (done) => {
+          const receiverPromise = receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const testSession = events.find(event => event.type === 'test_session_end').content
+              assert.equal(testSession.meta[TEST_STATUS], 'fail')
+            })
+
+          receiver.setSettings({ test_management: { enabled: true } })
+
+          childProcess = exec(
+            './node_modules/.bin/playwright test -c playwright.config.js exit-code-test.js',
+            {
+              cwd,
+              env: {
+                ...getCiVisAgentlessConfig(receiver.port),
+                PW_BASE_URL: `http://localhost:${webAppPort}`,
+                TEST_DIR: './ci-visibility/playwright-tests-exit-code'
+              },
+              stdio: 'pipe'
+            }
+          )
+
+          childProcess.on('exit', (exitCode) => {
+            assert.equal(exitCode, 1)
+            receiverPromise.then(() => done()).catch(done)
+          })
+        })
+      })
     }
   })
 })

packages/datadog-instrumentations/src/playwright.js

@@ -604,7 +604,9 @@ function runnerHook (runnerExport, playwrightVersion) {
         totalAttemptToFixFailedTestCount += testStatuses.filter(status => status === 'fail').length
       }
 
-      if (totalFailedTestCount === totalAttemptToFixFailedTestCount) {
+      if (totalFailedTestCount > 0 &&
+          totalAttemptToFixFailedTestCount > 0 &&
+          totalFailedTestCount === totalAttemptToFixFailedTestCount) {
         runAllTestsReturn = 'passed'
       }
     }