fix: no error thrown from rerun script (#4494)

* ENG-1059: Throw error

* ENG-1059: Write tests

* Revert changes to test for flakiness

* Re-apply changes

This reverts commit 6cba723.

Author: lin-brian-l

lib/command/run-rerun.js

@@ -17,10 +17,6 @@ module.exports = async function (test, options) {
   const testRoot = getTestRoot(configFile)
   createOutputDir(config, testRoot)
 
-  function processError(err) {
-    printError(err)
-    process.exit(1)
-  }
   const codecept = new Codecept(config, options)
 
   try {

lib/rerun.js

@@ -70,6 +70,7 @@ class CodeceptRerunner extends BaseCodecept {
       await this.runTests(test);
     } catch (e) {
       output.error(e.stack);
+      throw e;
     } finally {
       event.emit(event.all.result, this);
       event.emit(event.all.after, this);

test/data/sandbox/configs/run-rerun/codecept.conf.pass_all_test.js

@@ -0,0 +1,16 @@
+exports.config = {
+  tests: './*_ftest.js',
+  output: './output',
+  helpers: {
+    CustomHelper: {
+      require: './customHelper.js',
+    },
+  },
+  rerun: {
+    minSuccess: 3,
+    maxReruns: 3,
+  },
+  bootstrap: null,
+  mocha: {},
+  name: 'run-rerun',
+};

test/runner/run_rerun_test.js

@@ -83,7 +83,7 @@ describe('run-rerun command', () => {
     )
   })
 
-  it('should display success run if test was fail one time of two attepmts and 3 reruns', (done) => {
+  it('should display success run if test was fail one time of two attempts and 3 reruns', (done) => {
     exec(
       `FAIL_ATTEMPT=0  ${codecept_run_config('codecept.conf.fail_test.js', '@RunRerun - fail second test')} --debug`,
       (err, stdout) => {
@@ -96,4 +96,18 @@ describe('run-rerun command', () => {
       },
     )
   })
+
+  it('should throw exit code 1 if all tests were supposed to pass', (done) => {
+    exec(
+      `FAIL_ATTEMPT=0  ${codecept_run_config('codecept.conf.pass_all_test.js', '@RunRerun - fail second test')} --debug`,
+      (err, stdout) => {
+        expect(stdout).toContain('Process run 1 of max 3, success runs 1/3')
+        expect(stdout).toContain('Fail run 2 of max 3, success runs 1/3')
+        expect(stdout).toContain('Process run 3 of max 3, success runs 2/3')
+        expect(stdout).toContain('Flaky tests detected!')
+        expect(err.code).toBe(1)
+        done()
+      },
+    )
+  })
 })