implement repeats

Refs: Re-run a test multiple times #2332

Author: mmomtchev

docs/index.md

@@ -707,6 +707,31 @@ describe("retries", function () {
 });
 ```
 
+## Repeat Tests
+
+Tests can also be repeated when they pass. This feature can be used to test for leaks and proper tear-down procedures. In this case a test is considered to be successful only if all the runs are successful.
+
+This feature does re-run a passed test and its corresponding `beforeEach/afterEach` hooks, but not `before/after` hooks.
+
+If using both `repeat` and `retries`, the test will be run `repeat` times tolerating up to `retries` failures in total.
+
+```js
+describe('repeat', function () {
+  // Repeat all tests in this suite 4 times
+  this.repeats(4);
+
+  beforeEach(function () {
+    browser.get('http://www.yahoo.com');
+  });
+
+  it('should use proper tear-down', function () {
+    // Specify this test to only retry up to 2 times
+    this.repeats(2);
+    expect($('.foo').isDisplayed()).to.eventually.be.true;
+  });
+});
+```
+
 ## Dynamically Generating Tests
 
 Given Mocha's use of function expressions to define suites and test cases, it's straightforward to generate your tests dynamically. No special syntax is required — plain ol' JavaScript can be used to achieve functionality similar to "parameterized" tests, which you may have seen in other frameworks.
@@ -2218,6 +2243,7 @@ mocha.setup({
   forbidPending: true,
   global: ['MyLib'],
   retries: 3,
+  repeats: 1,
   rootHooks: { beforeEach(done) { ... done();} },
   slow: '100',
   timeout: '2000',

example/config/.mocharc.js

@@ -36,6 +36,7 @@ module.exports = {
   "reporter-option": ["foo=bar", "baz=quux"], // array, not object
   require: "@babel/register",
   retries: 1,
+  repeats: 1,
   slow: "75",
   sort: false,
   spec: ["test/**/*.spec.js"], // the positional arguments!

example/config/.mocharc.yml

@@ -40,6 +40,7 @@ reporter-option: # array, not object
   - "baz=quux"
 require: "@babel/register"
 retries: 1
+repeats: 1
 slow: "75"
 sort: false
 spec:

lib/cli/run-option-metadata.js

@@ -51,7 +51,7 @@ const TYPES = (exports.types = {
     "sort",
     "watch",
   ],
-  number: ["retries", "jobs"],
+  number: ["retries", "repeats", "jobs"],
   string: [
     "config",
     "fgrep",

lib/cli/run.js

@@ -247,6 +247,10 @@ exports.builder = (yargs) =>
         description: "Retry failed tests this many times",
         group: GROUPS.RULES,
       },
+      repeats: {
+        description: 'Repeat passed tests this many times',
+        group: GROUPS.RULES
+      },
       slow: {
         default: defaults.slow,
         description: 'Specify "slow" test threshold (in milliseconds)',

lib/context.js

@@ -89,3 +89,18 @@ Context.prototype.retries = function (n) {
   this.runnable().retries(n);
   return this;
 };
+
+/**
+ * Set or get a number of repeats on passed tests
+ *
+ * @private
+ * @param {number} n
+ * @return {Context} self
+ */
+Context.prototype.repeats = function (n) {
+  if (!arguments.length) {
+    return this.runnable().repeats();
+  }
+  this.runnable().repeats(n);
+  return this;
+};

lib/hook.js

@@ -63,6 +63,7 @@ Hook.prototype.error = function (err) {
 Hook.prototype.serialize = function serialize() {
   return {
     $$currentRetry: this.currentRetry(),
+    $$currentRepeat: this.currentRepeat(),
     $$fullTitle: this.fullTitle(),
     $$isPending: Boolean(this.isPending()),
     $$titlePath: this.titlePath(),

lib/mocha.js

@@ -188,6 +188,10 @@ function Mocha(options = {}) {
     this.retries(options.retries);
   }
 
+  if ("repeats" in options) {
+    this.repeats(options.repeats);
+  }
+
   [
     "allowUncaught",
     "asyncOnly",
@@ -745,6 +749,25 @@ Mocha.prototype.retries = function (retry) {
   return this;
 };
 
+/**
+ * Sets the number of times to repeat passed tests.
+ *
+ * @public
+ * @see [CLI option](../#-repeats-n)
+ * @see [Repeat Tests](../#repeat-tests)
+ * @param {number} repeats - Number of times to repeat passed tests.
+ * @return {Mocha} this
+ * @chainable
+ * @example
+ *
+ * // Allow any passed test to be repeated multiple times
+ * mocha.repeats(1);
+ */
+Mocha.prototype.repeats = function (repeats) {
+  this.suite.repeats(repeats);
+  return this;
+};
+
 /**
  * Sets slowness threshold value.
  *

lib/reporters/json-stream.js

@@ -86,6 +86,7 @@ function clean(test) {
     file: test.file,
     duration: test.duration,
     currentRetry: test.currentRetry(),
+    currentRepeat: test.currentRepeat(),
     speed: test.speed,
   };
 }

lib/reporters/json.js

@@ -120,6 +120,7 @@ function clean(test) {
     file: test.file,
     duration: test.duration,
     currentRetry: test.currentRetry(),
+    currentRepeat: test.currentRepeat(),
     speed: test.speed,
     err: cleanCycles(err),
   };