benchmark: add per-suite setup option

This allows us to set up fixtures for the benchmark suite only
once, which can save quite a bit of time when running benchmarks
that require tens of thousands of fixture files or more (e.g.
the module benchmarks).

Author: joyeecheung

benchmark/common.js

@@ -28,6 +28,7 @@ class Benchmark {
     const argv = process.argv.slice(2);
     const parsed_args = this._parseArgs(argv, configs, options);
 
+    this.originalOptions = options;
     this.options = parsed_args.cli;
     this.extra_options = parsed_args.extra;
     this.combinationFilter = typeof options.combinationFilter === 'function' ? options.combinationFilter : allow;
@@ -207,6 +208,12 @@ class Benchmark {
       });
     }
 
+    if (this.originalOptions.setup) {
+      // Only do this from the root process. _run() is only ever called from the root,
+      // in child processes main is run directly.
+      this.originalOptions.setup(this.queue);
+    }
+
     const recursive = (queueIndex) => {
       const config = this.queue[queueIndex];
 

benchmark/module/module-require.js

@@ -8,12 +8,15 @@ const benchmarkDirectory = tmpdir.resolve('nodejs-benchmark-module');
 const bench = common.createBenchmark(main, {
   type: ['.js', '.json', 'dir'],
   n: [1e4],
+}, {
+  setup(configs) {
+    tmpdir.refresh();
+    const maxN = configs.reduce((max, c) => Math.max(max, c.n), 0);
+    createEntryPoint(maxN);
+  },
 });
 
 function main({ type, n }) {
-  tmpdir.refresh();
-  createEntryPoint(n);
-
   switch (type) {
     case '.js':
       measureJSFile(n);
@@ -24,8 +27,6 @@ function main({ type, n }) {
     case 'dir':
       measureDir(n);
   }
-
-  tmpdir.refresh();
 }
 
 function measureJSFile(n) {

doc/contributing/writing-and-running-benchmarks.md

@@ -591,6 +591,29 @@ The arguments of `createBenchmark` are:
     containing a combination of benchmark parameters. It should return `true`
     or `false` to indicate whether the combination should be included or not.
 
+  * `setup` {Function} A function that will be run once before the benchmark
+    combinations are executed. It can be used to setup any global state
+    required by the benchmark. The argument passed into it is an array of all
+    the combinations of configurations that will be executed.
+    If tear down is necessary, register a listener for the `exit` event on
+    `process` inside the `setup` function. In the example below, that's done
+    by `tmpdir.refresh()`.
+
+    ```js
+    const tmpdir = require('../../test/common/tmpdir');
+    const bench = common.createBenchmark(main, {
+      type: ['fast', 'slow'],
+      n: [1e4],
+    }, {
+      setup(configs) {
+        tmpdir.refresh();
+        const maxN = configs.reduce((max, c) => Math.max(max, c.n), 0);
+        setupFixturesReusedForAllBenchmarks(maxN);
+      },
+    });
+    ```
+
+
 `createBenchmark` returns a `bench` object, which is used for timing
 the runtime of the benchmark. Run `bench.start()` after the initialization
 and `bench.end(n)` when the benchmark is done. `n` is the number of operations