benchmark: add --csv option to compare.js with --analyze

Add a `--csv {filename}` option to benchmark/compare.js
to capture the CSV when the `--analyze` option is used

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode

Author: jasnell

benchmark/compare.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { spawn, fork } = require('node:child_process');
+const { closeSync, openSync, writeSync } = require('node:fs');
 const { inspect } = require('util');
 const path = require('path');
 const CLI = require('./_cli.js');
@@ -27,7 +28,9 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
   --no-progress                 don't show benchmark progress indicator
   --analyze                     perform statistical analysis after benchmarks
                                 complete (Welch's t-test, effect size) instead
-                                of printing csv output
+                                of printing csv output to stdout
+  --csv      filename           write csv output to filename (can be combined
+                                with --analyze)
   --scale    1000               rate-to-integer multiplier for histogram
                                  precision when using --analyze (default: 1000)
   --max-regression  N           exit with code 1 if any statistically
@@ -60,6 +63,19 @@ if (benchmarks.length === 0) {
   return;
 }
 
+const csvFd = cli.optional.csv === undefined ?
+  null :
+  openSync(cli.optional.csv, 'w');
+const outputCsv = !analyze || csvFd !== null;
+
+function writeCsv(line) {
+  if (csvFd === null) {
+    console.log(line);
+  } else {
+    writeSync(csvFd, `${line}\n`);
+  }
+}
+
 // When --analyze is set, collect results for statistical analysis.
 const results = analyze ? new Map() : null;
 
@@ -78,17 +94,19 @@ for (const filename of benchmarks) {
 }
 // queue.length = binary.length * runs * benchmarks.length
 
-// Print csv header (unless analyzing inline).
-if (!analyze) {
-  console.log('"binary","filename","configuration","rate","time"');
+// Print csv header unless only analyzing inline.
+if (outputCsv) {
+  writeCsv('"binary","filename","configuration","rate","time"');
 }
 
 const kStartOfQueue = 0;
 
 const showProgress = !cli.optional['no-progress'];
 let progress;
 if (showProgress) {
-  progress = new BenchmarkProgress(queue, benchmarks, { analyze });
+  progress = new BenchmarkProgress(queue, benchmarks, {
+    analyze: analyze || csvFd !== null,
+  });
   progress.startQueue(kStartOfQueue);
 }
 
@@ -126,11 +144,13 @@ if (showProgress) {
           results.set(name, { old: [], new: [] });
         }
         results.get(name)[job.binary].push(data.rate);
-      } else {
+      }
+
+      if (outputCsv) {
         // Escape quotes (") for correct csv formatting
-        conf = conf.replace(/"/g, '""');
-        console.log(`"${job.binary}","${job.filename}","${conf}",` +
-                    `${data.rate},${data.time}`);
+        const csvConf = conf.replace(/"/g, '""');
+        writeCsv(`"${job.binary}","${job.filename}","${csvConf}",` +
+                 `${data.rate},${data.time}`);
       }
       if (showProgress) {
         // One item in the subqueue has been completed.
@@ -153,8 +173,9 @@ if (showProgress) {
     // If there are more benchmarks execute the next
     if (i + 1 < queue.length) {
       recursive(i + 1);
-    } else if (analyze) {
-      printAnalysis(results, scale, maxRegression);
+    } else {
+      if (csvFd !== null) closeSync(csvFd);
+      if (analyze) printAnalysis(results, scale, maxRegression);
     }
   });
 })(kStartOfQueue);

doc/contributing/writing-and-running-benchmarks.md

@@ -416,6 +416,8 @@ module, you can use the `--filter` option:_
   --set      variable=value     set benchmark variable (can be repeated)
   --no-progress                 don't show benchmark progress indicator
   --analyze                     perform statistical analysis inline (no R needed)
+  --csv      filename           write csv output to filename (can be combined
+                                with --analyze)
   --scale    1000               rate multiplier for --analyze precision
   --max-regression  N           exit with code 1 if any significant regression
                                 exceeds N% (implies --analyze)
@@ -429,6 +431,14 @@ The simplest way to get statistical results is to pass `--analyze`:
 node benchmark/compare.js --old ./node-main --new ./node-pr-5134 --analyze string_decoder
 ```
 
+Use `--csv` to retain the raw benchmark results while also printing the
+analysis:
+
+```bash
+node benchmark/compare.js --old ./node-main --new ./node-pr-5134 \
+  --analyze --csv compare-pr-5134.csv string_decoder
+```
+
 This runs the benchmarks and prints the analysis directly:
 
 ```console

test/parallel/test-benchmark-compare.js

@@ -0,0 +1,45 @@
+'use strict';
+
+require('../common');
+
+const assert = require('node:assert');
+const { spawnSync } = require('node:child_process');
+const { readFileSync } = require('node:fs');
+const path = require('node:path');
+const tmpdir = require('../common/tmpdir');
+
+const compare = path.resolve(__dirname, '../../benchmark/compare.js');
+
+tmpdir.refresh();
+
+const csv = tmpdir.resolve('compare.csv');
+const result = spawnSync(process.execPath, [
+  compare,
+  '--old', process.execPath,
+  '--new', process.execPath,
+  '--runs', '1',
+  '--filter', 'buffer-compare-offset.js',
+  '--set', 'method=offset',
+  '--set', 'size=16',
+  '--set', 'n=1',
+  '--no-progress',
+  '--analyze',
+  '--csv', csv,
+  'buffers',
+], {
+  encoding: 'utf8',
+  timeout: 30_000,
+});
+
+assert.strictEqual(result.status, 0, result.stderr);
+assert.strictEqual(result.stderr, '');
+assert.match(result.stdout, /confidence\s+improvement\s+accuracy/);
+assert.doesNotMatch(result.stdout, /"binary","filename"/);
+
+const lines = readFileSync(csv, 'utf8').trim().split('\n');
+const filename = path.join('buffers', 'buffer-compare-offset.js');
+assert.strictEqual(lines[0],
+                   '"binary","filename","configuration","rate","time"');
+assert.strictEqual(lines.length, 3);
+assert(lines[1].startsWith(`"old","${filename}",`));
+assert(lines[2].startsWith(`"new","${filename}",`));