Retry befores (#3580)

* retry for hooks initial

* retry for before & beforeSuite

* updated tests, refactored timeouts

* fixed tests for BDD

* fixed bdd test

* a better hook for a tes

* updated docs for timeouts and retries

* fixed examples

* simplified test

Author: DavertMik

docs/advanced.md

@@ -186,7 +186,7 @@ You can use this options for build your own [plugins](https://codecept.io/hooks/
   });
 ```
 
-## Timeout <Badge text="Updated in 3.2" type="warning"/>
+## Timeout
 
 Tests can get stuck due to various reasons such as network connection issues, crashed browser, etc.
 This can make tests process hang. To prevent these situations timeouts can be used. Timeouts can be set explicitly for flaky parts of code, or implicitly in a config.
@@ -236,38 +236,60 @@ A timeout for a group of tests can be set on Feature level via options.
 Feature('flaky tests', { timeout: 30 })
 ```
 
-### Sum Up
+### Timeout Confguration <Badge text="Updated in 3.4" type="warning"/>
 
-Let's list all available timeout options.
+Timeout rules can be set globally via config.
 
-Timeouts can be set globally in config:
+To set a timeout for all running tests provide a **number of seconds** to `timeout` config option:
 
-```js
-// in codecept.confg.js:
-{ // ...
-   timeout: 30, // limit all tests in all suites to 30 secs
-
-   plugins: {
-     stepTimeout: {
-       enabled: true,
-       timeout: 10, // limit all steps except waiters to 10 secs
-     }
-   }
-} 
 
+```js
+// inside codecept.conf.js or codecept.conf.ts
+timeout: 30, // limit all tests in all suites to 30 secs
 ```
 
-or inside a test file:
+It is possible to tune this configuration for a different groups of tests passing options as array and using `grep` option to filter tests:
 
 ```js
-// limit all tests in this suite to 10 secs
-Feature('tests with timeout', { timeout: 10 });
+// inside codecept.conf.js or codecept.conf.ts
 
-// limit this test to 20 secs
-Scenario('a test with timeout', { timeout: 20 }, ({ I }) => {
-   // limit step to 5 seconds
-   I.limitTime(5).click('Link');
-});
+timeout: [
+  10, // default timeout is 10secs  
+
+  // but increase timeout for slow tests
+  {
+    grep: '@slow',
+    Feature: 50
+  },
+]
+```
+
+> ℹ️ `grep` value can be string or regexp
+
+It is possible to set a timeout for Scenario or Feature:
+
+```js
+// inside codecept.conf.js or codecept.conf.ts
+timeout: [
+
+  // timeout for Feature with @slow in title
+  {
+    grep: '@slow',
+    Feature: 50
+  },
+  
+  // timeout for Scenario with 'flaky0' .. `flaky1` in title
+  {
+    // regexp can be passed to grep
+    grep: /flaky[0-9]/,
+    Scenario: 10
+  },
+
+  // timeout for all suites
+  {    
+    Feature: 20
+  }
+]
 ```
 
 Global timeouts will be overridden by explicit timeouts of a test or steps.

docs/basics.md

@@ -345,7 +345,7 @@ If you face that error please make sure that all async functions are called with
 
 ## Running Tests
 
-To launch tests use the `run` command, and to execute tests in [multiple browsers](/advanced/#multiple-browsers-execution) or [multiple threads](/advanced/#parallel-execution) use the `run-multiple` command.
+To launch tests use the `run` command, and to execute tests in [multiple threads](/advanced/parallel) using `run-workers` command.
 
 ### Level of Detail
 
@@ -394,7 +394,7 @@ It is recommended to [filter tests by tags](/advanced/#tags).
 
 ### Parallel Run
 
-Since CodeceptJS 2.3, you can run tests in parallel by using NodeJS workers. This feature requires NodeJS >= 11.6. Use `run-workers` command with the number of workers (threads) to split tests.
+Tests can be executed in parallel mode by using [NodeJS workers](https://nodejs.org/api/worker_threads.html). Use `run-workers` command with the number of workers (threads) to split tests into different workers.
 
 ```
 npx codeceptjs run-workers 3
@@ -532,13 +532,55 @@ This can be configured in [screenshotOnFail Plugin](/plugins/#screenshotonfail)
 
 To see how the test was executed, use [stepByStepReport Plugin](/plugins/#stepbystepreport). It saves a screenshot of each passed step and shows them in a nice slideshow.
 
+## Before
+
+Common preparation steps like opening a web page or logging in a user, can be placed in the `Before` or `Background` hooks:
+
+```js
+Feature('CodeceptJS Demonstration');
+
+Before(({ I }) => { // or Background
+  I.amOnPage('/documentation');
+});
+
+Scenario('test some forms', ({ I }) => {
+  I.click('Create User');
+  I.see('User is valid');
+  I.dontSeeInCurrentUrl('/documentation');
+});
+
+Scenario('test title', ({ I }) => {
+  I.seeInTitle('Example application');
+});
+```
+
+Same as `Before` you can use `After` to run teardown for each scenario.
+
+## BeforeSuite
+
+If you need to run complex a setup before all tests and have to teardown this afterwards, you can use the `BeforeSuite` and `AfterSuite` functions.
+`BeforeSuite` and `AfterSuite` have access to the `I` object, but `BeforeSuite/AfterSuite` don't have any access to the browser, because it's not running at this moment.
+You can use them to execute handlers that will setup your environment. `BeforeSuite/AfterSuite` will work only for the file it was declared in (so you can declare different setups for files)
+
+```js
+BeforeSuite(({ I }) => {
+  I.syncDown('testfolder');
+});
+
+AfterSuite(({ I }) => {
+  I.syncUp('testfolder');
+  I.clearDir('testfolder');
+});
+```
+
 ## Retries
 
 ### Auto Retry
 
-You can auto-retry a failed step by enabling [retryFailedStep Plugin](/plugins/#retryfailedstep).
+Each failed step is auto-retried by default via [retryFailedStep Plugin](/plugins/#retryfailedstep).
+If this is not expected, this plugin can be disabled in a config. 
 
-> **[retryFailedStep plugin](/plugins/#retryfailedstep) is enabled by default** for new setups
+> **[retryFailedStep plugin](/plugins/#retryfailedstep) is enabled by default** incide global configuration
 
 ### Retry Step
 
@@ -608,69 +650,102 @@ Scenario('Really complex', { retries: 2 },({ I }) => {});
 This scenario will be restarted two times on a failure.
 Unlike retry step, there is no `when` condition supported for retries on a scenario level.
 
+### Retry Before <Badge text="Since 3.4" type="warning"/>
+
+To retry `Before`, `BeforeSuite`, `After`, `AfterSuite` hooks, add corresponding option to a `Feature`:
+
+* `retryBefore`
+* `retryBeforeSuite`
+* `retryAfter`
+* `retryAfterSuite`
+
+For instance, to retry Before hook 3 times:
+
+```js
+Feature('this have a flaky Befure', { retryBefore: 3 })
+```
+
+Multiple options of different values can be set at the same time
+
 ### Retry Feature
 
 To set this option for all scenarios in a file, add `retry` to a feature:
 
 ```js
 Feature('Complex JS Stuff').retry(3);
+// or
+Feature('Complex JS Stuff', { retries: 3 })
 ```
 
 Every Scenario inside this feature will be rerun 3 times.
 You can make an exception for a specific scenario by passing the `retries` option to a Scenario.
 
-### Retry Run 
-
-On the highest level of the "retry pyramid" there is an option to retry a complete run multiple times.
-Even this is the slowest option of all, it can be helpful to detect flaky tests. 
+### Retry Configuration <Badge text="Since 3.4" type="warning"/>
 
-[`run-rerun`](https://codecept.io/commands/#run-rerun) command will restart the run multiple times to values you provide. You can set minimal and maximal number of restarts in configuration file.
+It is possible to set retry rules globally via `retry` config option. The configuration is flexible and allows multiple formats.
+The simplest config would be:
 
+```js
+// inside codecept.conf.js
+retry: 3
 ```
-npx codeceptjs run-rerun
-```
-
 
-## Before
+This will enable Feature Retry for all executed feature, retrying failing tests 3 times.
 
-Common preparation steps like opening a web page or logging in a user, can be placed in the `Before` or `Background` hooks:
+An object can be used to tune retries of a Before/After hook, Scenario or Feature
 
 ```js
-Feature('CodeceptJS Demonstration');
+// inside codecept.conf.js
+retry: {
+  Feature: ...,
+  Scenario: ...,
+  Before: ...,
+  BeforeSuite: ...,
+  After: ...,
+  AfterSuite: ..., 
+}
+```
 
-Before(({ I }) => { // or Background
-  I.amOnPage('/documentation');
-});
+Multiple retry configs can be added via array. To use different retry configs for different subset of tests use `grep` option inside a retry config:
+
+```js
+// inside codecept.conf.js
+retry: [
+  {
+    // enable this config only for flaky tests
+    grep: '@flaky', 
+    Before: 3
+    Scenario: 3
+  }, 
+  {
+    // retry less when running slow tests
+    grep: '@slow' 
+    Scenario: 1
+    Before: 1
+  }, {
+    // retry all BeforeSuite 
+    BeforeSuite: 3
+  }
+]
+```
 
-Scenario('test some forms', ({ I }) => {
-  I.click('Create User');
-  I.see('User is valid');
-  I.dontSeeInCurrentUrl('/documentation');
-});
+When using `grep` with `Before`, `After`, `BeforeSuite`, `AfterSuite`, a suite title will be checked for included value. 
 
-Scenario('test title', ({ I }) => {
-  I.seeInTitle('Example application');
-});
-```
+> ℹ️ `grep` value can be string or regexp
 
-Same as `Before` you can use `After` to run teardown for each scenario.
+Rules are applied in the order of array element, so the last option will override a previous one. Global retries config can be overridden in a file as described previously.
 
-## BeforeSuite
+### Retry Run 
 
-If you need to run complex a setup before all tests and have to teardown this afterwards, you can use the `BeforeSuite` and `AfterSuite` functions.
-`BeforeSuite` and `AfterSuite` have access to the `I` object, but `BeforeSuite/AfterSuite` don't have any access to the browser, because it's not running at this moment.
-You can use them to execute handlers that will setup your environment. `BeforeSuite/AfterSuite` will work only for the file it was declared in (so you can declare different setups for files)
+On the highest level of the "retry pyramid" there is an option to retry a complete run multiple times.
+Even this is the slowest option of all, it can be helpful to detect flaky tests. 
 
-```js
-BeforeSuite(({ I }) => {
-  I.syncDown('testfolder');
-});
+[`run-rerun`](https://codecept.io/commands/#run-rerun) command will restart the run multiple times to values you provide. You can set minimal and maximal number of restarts in configuration file.
 
-AfterSuite(({ I }) => {
-  I.syncUp('testfolder');
-  I.clearDir('testfolder');
-});
 ```
+npx codeceptjs run-rerun
+```
+
 
 [Here are some ideas](https://github.com/codeceptjs/CodeceptJS/pull/231#issuecomment-249554933) on where to use BeforeSuite hooks.
 

examples/checkout_test.js

@@ -1,14 +1,17 @@
-Feature('Checkout');
+Feature('Checkout', { retryBefore: 3 });
+
+const i = 0;
 
 Before(({ I }) => {
   I.amOnPage('https://getbootstrap.com/docs/4.0/examples/checkout/');
+  I.failNTimes(3);
 });
 
 Scenario('It should fill in checkout page', async ({ I }) => {
   I.fillField('#lastName', 'mik');
-  await eachElement('tick all checkboxes', '.custom-checkbox label', async (el) => {
-    await el.click();
-  });
+  // await eachElement('tick all checkboxes', '.custom-checkbox label', async (el) => {
+  //   await el.click();
+  // });
   await retryTo((retryNum) => {
     I.fillField('Promo code', '123345');
     I.click('Redeem');

examples/codecept.config.js

@@ -1,5 +1,6 @@
 exports.config = {
   output: './output',
+  retry: 3,
   helpers: {
     Playwright: {
       url: 'http://localhost',
@@ -14,6 +15,9 @@ exports.config = {
       show: !process.env.HEADLESS,
     },
     REST: {},
+    User: {
+      require: './user_helper.js',
+    },
   },
   include: {
     I: './custom_steps.js',
@@ -60,7 +64,7 @@ exports.config = {
     },
   },
   tests: './*_test.js',
-  timeout: 100,
+  // timeout: 100,
   multiple: {
     parallel: {
       chunks: 2,

examples/user_helper.js

@@ -1,14 +1,18 @@
 const assert = require('assert');
-const Helper = require('../lib/helper');
 
 class User extends Helper {
-  // before/after hooks
+  _beforeSuite() {
+  }
+
   _before() {
-    // remove if not used
+    this.i = 0;
   }
 
-  _after() {
-    // remove if not used
+  failNTimes(n) {
+    this.i++;
+    // this.i++;
+    console.log(this.i, n);
+    if (this.i < n) throw new Error('ups, error');
   }
 
   // add custom methods here

lib/codecept.js

@@ -98,6 +98,7 @@ class Codecept {
     runHook(require('./listener/artifacts'));
     runHook(require('./listener/config'));
     runHook(require('./listener/helpers'));
+    runHook(require('./listener/retry'));
     runHook(require('./listener/timeout'));
     runHook(require('./listener/exit'));
 

lib/interfaces/featureConfig.js

@@ -19,6 +19,7 @@ class FeatureConfig {
    * Set timeout for this suite
    * @param {number} timeout
    * @returns {this}
+   * @deprecated
    */
   timeout(timeout) {
     console.log(`Feature('${this.suite.title}').timeout(${timeout}) is deprecated!`);